safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

more format changes for Mr. Travis

Browse Source
This commit is contained in:
ThomasBarris
2018-08-31 22:01:53 +02:00
parent 42e79bfab9
commit aa6ac5a751
1 changed files with 147 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

404
lib/SetupClass.php
View File

@@ -4,28 +4,29 @@ namespace Nominatim\Setup;
class SetupFunctions
{
protected $iCacheMemory; // set in constructor
protected $iInstances; // set in constructor
protected $sModulePath; // set in constructor
protected $aDSNInfo; // set in constructor = DB::parseDSN(CONST_Database_DSN);
protected $sVerbose; // set in constructor
protected $sIgnoreErrors; // set in constructor
protected $bEnableDiffUpdates; // set in constructor
protected $bEnableDebugStatements; // set in constructor
protected $bNoPartitions; // set in constructor
protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData
// pgsqlRunPartitionScript, calculatePostcodes, ..if no already set
protected $iCacheMemory; // set in constructor
protected $iInstances; // set in constructor
protected $sModulePath; // set in constructor
protected $aDSNInfo; // set in constructor = DB::parseDSN(CONST_Database_DSN);
protected $sVerbose; // set in constructor
protected $sIgnoreErrors; // set in constructor
protected $bEnableDiffUpdates; // set in constructor
protected $bEnableDebugStatements; // set in constructor
protected $bNoPartitions; // set in constructor
protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData
// pgsqlRunPartitionScript, calculatePostcodes, ..if no already set
public function __construct($aCMDResult)
{
// by default, use all but one processor, but never more than 15.
$this->iInstances = isset($aCMDResult['threads'])
? $aCMDResult['threads']
: (min(16, getProcessorCount()) - 1);
? $aCMDResult['threads']
: (min(16, getProcessorCount()) - 1);
if ($this->iInstances < 1) {
$this->iInstances = 1;
warn('resetting threads to ' . $this->iInstances);
warn('resetting threads to '.$this->iInstances);
}
// Assume we can steal all the cache memory in the box (unless told otherwise)
@@ -37,13 +38,11 @@ class SetupFunctions
$this->sModulePath = CONST_Database_Module_Path;
info('module path: ' . $this->sModulePath);
// prepares DB for import or update, sets the Data Source Name
$this->aDSNInfo = \DB::parseDSN(CONST_Database_DSN);
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) {
$this->aDSNInfo['port'] = 5432;
}
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
// setting member variables based on command line options stored in $aCMDResult
$this->sVerbose = $aCMDResult['verbose'];
$this->sIgnoreErrors = $aCMDResult['ignore-errors'];
@@ -57,10 +56,10 @@ class SetupFunctions
info('Create DB');
$sDB = \DB::connect(CONST_Database_DSN, false);
if (!\PEAR::isError($sDB)) {
fail('database already exists (' . CONST_Database_DSN . ')');
fail('database already exists ('.CONST_Database_DSN.')');
}
$sCreateDBCmd = 'createdb -E UTF-8 -p ' . $this->aDSNInfo['port'] . ' ' . $this->aDSNInfo['database'];
$sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database'];
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username'];
}
@@ -75,19 +74,16 @@ class SetupFunctions
}
$result = runWithEnv($sCreateDBCmd, $aProcEnv);
if ($result != 0) {
fail('Error executing external command: ' . $sCreateDBCmd);
}
if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd);
}
public function setupDB()
{
info('Setup DB');
$this->oDB = &getDB();
$this->oDB =& getDB();
$fPostgresVersion = getPostgresVersion($this->oDB);
echo 'Postgres version found: ' . $fPostgresVersion . "\n";
echo 'Postgres version found: '.$fPostgresVersion."\n";
if ($fPostgresVersion < 9.1) {
fail('Minimum supported version of Postgresql is 9.1.');
@@ -106,8 +102,9 @@ class SetupFunctions
warn('Postgresql is too old. extratags and namedetails API not available.');
}
$fPostgisVersion = getPostgisVersion($this->oDB);
echo 'Postgis version found: ' . $fPostgisVersion . "\n";
echo 'Postgis version found: '.$fPostgisVersion."\n";
if ($fPostgisVersion < 2.1) {
// Functions were renamed in 2.1 and throw an annoying deprecation warning
@@ -118,31 +115,32 @@ class SetupFunctions
$this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
}
$i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '" . CONST_Database_Web_User . "'"));
$i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'"));
if ($i == 0) {
echo "\nERROR: Web user '" . CONST_Database_Web_User . "' does not exist. Create it with:\n";
echo "\n createuser " . CONST_Database_Web_User . "\n\n";
echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
echo "\n createuser ".CONST_Database_Web_User."\n\n";
exit(1);
}
if (!file_exists(CONST_ExtraDataPath . '/country_osm_grid.sql.gz')) {
if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) {
echo 'Error: you need to download the country_osm_grid first:';
echo "\n wget -O " . CONST_ExtraDataPath . "/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n";
echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n";
exit(1);
}
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_name.sql');
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_naturalearthdata.sql');
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_osm_grid.sql.gz');
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_table.sql');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
if (file_exists(CONST_BasePath . '/data/gb_postcode_data.sql.gz')) {
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_data.sql.gz');
if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) {
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
} else {
warn('external UK postcode table not found.');
}
if (CONST_Use_Extra_US_Postcodes) {
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/us_postcode.sql');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
}
if ($this->bNoPartitions) {
@@ -168,29 +166,23 @@ class SetupFunctions
fail("osm2pgsql not found in '$osm2pgsql'");
}
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
$osm2pgsql .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File;
}
if (CONST_Tablespace_Osm2pgsql_Data) {
$osm2pgsql .= ' --tablespace-slim-data ' . CONST_Tablespace_Osm2pgsql_Data;
}
if (CONST_Tablespace_Osm2pgsql_Index) {
$osm2pgsql .= ' --tablespace-slim-index ' . CONST_Tablespace_Osm2pgsql_Index;
}
if (CONST_Tablespace_Place_Data) {
$osm2pgsql .= ' --tablespace-main-data ' . CONST_Tablespace_Place_Data;
}
if (CONST_Tablespace_Place_Index) {
$osm2pgsql .= ' --tablespace-main-index ' . CONST_Tablespace_Place_Index;
$osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
}
if (CONST_Tablespace_Osm2pgsql_Data)
$osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data;
if (CONST_Tablespace_Osm2pgsql_Index)
$osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index;
if (CONST_Tablespace_Place_Data)
$osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data;
if (CONST_Tablespace_Place_Index)
$osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index;
$osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
$osm2pgsql .= ' -C ' . $this->iCacheMemory;
$osm2pgsql .= ' -P ' . $this->aDSNInfo['port'];
$osm2pgsql .= ' -C '.$this->iCacheMemory;
$osm2pgsql .= ' -P '.$this->aDSNInfo['port'];
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$osm2pgsql .= ' -U ' . $this->aDSNInfo['username'];
}
@@ -201,12 +193,9 @@ class SetupFunctions
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
}
$osm2pgsql .= ' -d ' . $this->aDSNInfo['database'] . ' ' . $sOSMFile;
$osm2pgsql .= ' -d '.$this->aDSNInfo['database'].' '.$sOSMFile;
runWithEnv($osm2pgsql, $aProcEnv);
if ($this->oDB == null) {
$this->oDB = &getDB();
}
if ($this->oDB == null) $this->oDB =& getDB();
if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) {
fail('No Data');
}
@@ -223,7 +212,7 @@ class SetupFunctions
{
info('Create Tables');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/tables.sql');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:address-data}',
@@ -263,7 +252,7 @@ class SetupFunctions
{
info('Create Partition Tables');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-tables.src.sql');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
$sTemplate = $this->replaceTablespace(
'{ts:address-data}',
CONST_Tablespace_Address_Data,
@@ -307,14 +296,14 @@ class SetupFunctions
{
info('Create Partition Functions');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-functions.src.sql');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
$this->pgsqlRunPartitionScript($sTemplate);
}
public function importWikipediaArticles()
{
$sWikiArticlesFile = CONST_Wikipedia_Data_Path . '/wikipedia_article.sql.bin';
$sWikiRedirectsFile = CONST_Wikipedia_Data_Path . '/wikipedia_redirect.sql.bin';
$sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
$sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
if (file_exists($sWikiArticlesFile)) {
info('Importing wikipedia articles');
$this->pgsqlRunDropAndRestore($sWikiArticlesFile);
@@ -334,78 +323,40 @@ class SetupFunctions
{
info('Drop old Data');
if ($this->oDB == null) {
$this->oDB = &getDB();
}
if (!pg_query($this->oDB->connection, 'TRUNCATE word')) {
fail(pg_last_error($this->oDB->connection));
}
if ($this->oDB == null) $this->oDB =& getDB();
if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection));
echo '.';
$sSQL = 'select distinct partition from country_name';
$aPartitions = chksql($this->oDB->getCol($sSQL));
if (!$this->bNoPartitions) {
$aPartitions[] = 0;
}
if (!$this->bNoPartitions) $aPartitions[] = 0;
foreach ($aPartitions as $sPartition) {
if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_' . $sPartition)) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection));
echo '.';
}
// used by getorcreate_word_id to ignore frequent partial words
$sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
$sSQL .= '$$ SELECT ' . CONST_Max_Word_Frequency . ' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
$sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
@@ -414,7 +365,7 @@ class SetupFunctions
// pre-create the word list
if (!$bDisableTokenPrecalc) {
info('Loading word list');
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/words.sql');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
}
info('Load Data');
@@ -422,30 +373,24 @@ class SetupFunctions
$aDBInstances = array();
$iLoadThreads = max(1, $this->iInstances - 1);
for ($i = 0; $i < $iLoadThreads; $i++) {
$aDBInstances[$i] = &getDB(true);
$aDBInstances[$i] =& getDB(true);
$sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
$sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
$sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')";
$sSQL .= ' and ST_IsValid(geometry)';
if ($this->sVerbose) {
echo "$sSQL\n";
}
if ($this->sVerbose) echo "$sSQL\n";
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
fail(pg_last_error($aDBInstances[$i]->connection));
}
}
// last thread for interpolation lines
$aDBInstances[$iLoadThreads] = &getDB(true);
$aDBInstances[$iLoadThreads] =& getDB(true);
$sSQL = 'insert into location_property_osmline';
$sSQL .= ' (osm_id, address, linegeo)';
$sSQL .= ' SELECT osm_id, address, geometry from place where ';
$sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";
if ($this->sVerbose) {
echo "$sSQL\n";
}
if ($this->sVerbose) echo "$sSQL\n";
if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) {
fail(pg_last_error($aDBInstances[$iLoadThreads]->connection));
}
@@ -477,7 +422,7 @@ class SetupFunctions
if ($sDatabaseDate === false) {
warn('could not determine database date.');
} else {
$sSQL = "INSERT INTO import_status (lastimportdate) VALUES('" . $sDatabaseDate . "')";
$sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
pg_query($this->oDB->connection, $sSQL);
echo "Latest data imported from $sDatabaseDate.\n";
}
@@ -487,7 +432,7 @@ class SetupFunctions
{
info('Import Tiger data');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_start.sql');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:aux-data}',
@@ -503,11 +448,11 @@ class SetupFunctions
$aDBInstances = array();
for ($i = 0; $i < $this->iInstances; $i++) {
$aDBInstances[$i] = &getDB(true);
$aDBInstances[$i] =& getDB(true);
}
foreach (glob(CONST_Tiger_Data_Path . '/*.sql') as $sFile) {
echo $sFile . ': ';
foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) {
echo $sFile.': ';
$hFile = fopen($sFile, 'r');
$sSQL = fgets($hFile, 100000);
$iLines = 0;
@@ -516,14 +461,8 @@ class SetupFunctions
if (!pg_connection_busy($aDBInstances[$i]->connection)) {
while (pg_get_result($aDBInstances[$i]->connection));
$sSQL = fgets($hFile, 100000);
if (!$sSQL) {
break 2;
}
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
if (!$sSQL) break 2;
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
$iLines++;
if ($iLines == 1000) {
echo '.';
@@ -539,10 +478,7 @@ class SetupFunctions
while ($bAnyBusy) {
$bAnyBusy = false;
for ($i = 0; $i < $this->iInstances; $i++) {
if (pg_connection_busy($aDBInstances[$i]->connection)) {
$bAnyBusy = true;
}
if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
}
usleep(10);
}
@@ -550,7 +486,7 @@ class SetupFunctions
}
info('Creating indexes on Tiger data');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_finish.sql');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:aux-data}',
@@ -568,15 +504,13 @@ class SetupFunctions
public function calculatePostcodes($bCMDResultAll)
{
info('Calculate Postcodes');
if ($this->oDB == null) {
$this->oDB = &getDB();
}
if ($this->oDB == null) $this->oDB =& getDB();
if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) {
fail(pg_last_error($this->oDB->connection));
}
$sSQL = 'INSERT INTO location_postcode';
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, country_code,";
$sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,";
@@ -592,29 +526,24 @@ class SetupFunctions
if (CONST_Use_Extra_US_Postcodes) {
// only add postcodes that are not yet available in OSM
$sSQL = 'INSERT INTO location_postcode';
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
$sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)';
$sSQL .= ' FROM us_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'us')";
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
}
// add missing postcodes for GB (if available)
$sSQL = 'INSERT INTO location_postcode';
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry";
$sSQL .= ' FROM gb_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'gb')";
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
if (!$bCMDResultAll) {
$sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
@@ -634,8 +563,8 @@ class SetupFunctions
public function index($bIndexNoanalyse)
{
$sOutputFile = '';
$sBaseCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $this->aDSNInfo['database'] . ' -P '
. $this->aDSNInfo['port'] . ' -t ' . $this->iInstances . $sOutputFile;
$sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P '
.$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec'];
}
@@ -648,45 +577,33 @@ class SetupFunctions
}
info('Index ranks 0 - 4');
$iStatus = runWithEnv($sBaseCmd . ' -R 4', $aProcEnv);
$iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv);
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
if (!$bIndexNoanalyse) {
$this->pgsqlRunScript('ANALYSE');
}
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
info('Index ranks 5 - 25');
$iStatus = runWithEnv($sBaseCmd . ' -r 5 -R 25', $aProcEnv);
$iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv);
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
if (!$bIndexNoanalyse) {
$this->pgsqlRunScript('ANALYSE');
}
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
info('Index ranks 26 - 30');
$iStatus = runWithEnv($sBaseCmd . ' -r 26', $aProcEnv);
$iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv);
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
info('Index postcodes');
if ($this->oDB == null) {
$this->oDB = &getDB();
}
if ($this->oDB == null) $this->oDB =& getDB();
$sSQL = 'UPDATE location_postcode SET indexed_status = 0';
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
}
public function createSearchIndices()
{
info('Create Search indices');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/indices.src.sql');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:address-index}',
@@ -715,12 +632,12 @@ class SetupFunctions
$this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x');
$this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x");
$sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),'
. 'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k ';
.'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k ';
if (CONST_Languages) {
$sSQL .= 'in ';
$sDelim = '(';
foreach (explode(',', CONST_Languages) as $sLang) {
$sSQL .= $sDelim . "'name:$sLang'";
$sSQL .= $sDelim."'name:$sLang'";
$sDelim = ',';
}
$sSQL .= ')';
@@ -742,27 +659,24 @@ class SetupFunctions
// created. USE AT YOUR OWN PERIL.
// tables we want to keep. everything else goes.
$aKeepTables = array(
'*columns',
'import_polygon_*',
'import_status',
'place_addressline',
'location_postcode',
'location_property*',
'placex',
'search_name',
'seq_*',
'word',
'query_log',
'new_query_log',
'spatial_ref_sys',
'country_name',
'place_classtype_*',
);
if ($this->oDB = null) {
$this->oDB = &getDB();
}
'*columns',
'import_polygon_*',
'import_status',
'place_addressline',
'location_postcode',
'location_property*',
'placex',
'search_name',
'seq_*',
'word',
'query_log',
'new_query_log',
'spatial_ref_sys',
'country_name',
'place_classtype_*'
);
if ($this->oDB = null) $this->oDB =& getDB();
$aDropTables = array();
$aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"));
@@ -774,37 +688,25 @@ class SetupFunctions
break;
}
}
if (!$bFound) {
array_push($aDropTables, $sTable);
}
if (!$bFound) array_push($aDropTables, $sTable);
}
foreach ($aDropTables as $sDrop) {
if ($this->sVerbose) {
echo "dropping table $sDrop\n";
}
if ($this->sVerbose) echo "dropping table $sDrop\n";
@pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE");
// ignore warnings/errors as they might be caused by a table having
// been deleted already by CASCADE
}
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
if ($sVerbose) {
echo 'deleting ' . CONST_Osm2pgsql_Flatnode_File . "\n";
}
if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n";
unlink(CONST_Osm2pgsql_Flatnode_File);
}
}
private function pgsqlRunDropAndRestore($sDumpFile)
{
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) {
$this->aDSNInfo['port'] = 5432;
}
$sCMD = 'pg_restore -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database'] . ' -Fc --clean ' . $sDumpFile;
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
$sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sCMD .= ' -h ' . $this->aDSNInfo['hostspec'];
}
@@ -815,9 +717,9 @@ class SetupFunctions
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
}
$iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)"
$iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)"
}
private function pgsqlRunScript($sScript, $bfatal = true)
{
runSQLScript(
@@ -830,7 +732,7 @@ class SetupFunctions
private function createSqlFunctions()
{
$sTemplate = file_get_contents(CONST_BasePath . '/sql/functions.sql');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
$sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
if ($this->bEnableDiffUpdates) {
$sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
@@ -852,15 +754,11 @@ class SetupFunctions
private function pgsqlRunPartitionScript($sTemplate)
{
if ($this->oDB == null) {
$this->oDB = &getDB();
}
if ($this->oDB == null) $this->oDB =& getDB();
$sSQL = 'select distinct partition from country_name';
$aPartitions = chksql($this->oDB->getCol($sSQL));
if (!$this->bNoPartitions) {
$aPartitions[] = 0;
}
if (!$this->bNoPartitions) $aPartitions[] = 0;
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
foreach ($aMatches as $aMatch) {
@@ -876,11 +774,9 @@ class SetupFunctions
private function pgsqlRunScriptFile($sFilename)
{
if (!file_exists($sFilename)) {
fail('unable to find ' . $sFilename);
}
if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
$sCMD = 'psql -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database'];
$sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'];
if (!$this->sVerbose) {
$sCMD .= ' -q';
}
@@ -897,32 +793,26 @@ class SetupFunctions
$ahGzipPipes = null;
if (preg_match('/\\.gz$/', $sFilename)) {
$aDescriptors = array(
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a'),
);
$hGzipProcess = proc_open('zcat ' . $sFilename, $aDescriptors, $ahGzipPipes);
if (!is_resource($hGzipProcess)) {
fail('unable to start zcat');
}
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a')
);
$hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes);
if (!is_resource($hGzipProcess)) fail('unable to start zcat');
$aReadPipe = $ahGzipPipes[1];
fclose($ahGzipPipes[0]);
} else {
$sCMD .= ' -f ' . $sFilename;
$sCMD .= ' -f '.$sFilename;
$aReadPipe = array('pipe', 'r');
}
$aDescriptors = array(
0 => $aReadPipe,
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a'),
);
0 => $aReadPipe,
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a')
);
$ahPipes = null;
$hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
if (!is_resource($hProcess)) {
fail('unable to start pgsql');
}
if (!is_resource($hProcess)) fail('unable to start pgsql');
// TODO: error checking
while (!feof($ahPipes[1])) {
echo fread($ahPipes[1], 4096);
@@ -941,7 +831,7 @@ class SetupFunctions
private function replaceTablespace($sTemplate, $sTablespace, $sSql)
{
if ($sTablespace) {
$sSql = str_replace($sTemplate, 'TABLESPACE "' . $sTablespace . '"', $sSql);
$sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql);
} else {
$sSql = str_replace($sTemplate, '', $sSql);
}