hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-07 10:34:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

timestamp info message and repeat warnings at end

Browse Source
This commit is contained in:
Sarah Hoffmann
2017-09-17 11:06:52 +02:00
parent cf32da3748
commit 3546b30473
2 changed files with 58 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
lib/cmd.php
View File

@@ -128,3 +128,23 @@ function chksql($oSql, $sMsg = false)
return $oSql; return $oSql;
} }
function info($sMsg)
{
echo date('Y-m-d H:i:s == ').$sMsg."\n";
}
$aWarnings = [];
function warn($sMsg)
{
$GLOBALS['aWarnings'][] = $sMsg;
echo date('Y-m-d H:i:s == ').'WARNING: '.$sMsg."\n";
}
function repeatWarnings()
{
foreach ($GLOBALS['aWarnings'] as $sMsg) {
echo ' * ',$sMsg."\n";
}
}

69
utils/setup.php
View File

@@ -65,11 +65,11 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) {
$iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1); $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
if ($iInstances < 1) { if ($iInstances < 1) {
$iInstances = 1; $iInstances = 1;
echo "WARNING: resetting threads to $iInstances\n"; warn("resetting threads to $iInstances");
} }
if ($iInstances > getProcessorCount()) { if ($iInstances > getProcessorCount()) {
$iInstances = getProcessorCount(); $iInstances = getProcessorCount();
echo "WARNING: resetting threads to $iInstances\n"; warn("resetting threads to $iInstances");
} }
// Assume we can steal all the cache memory in the box (unless told otherwise) // Assume we can steal all the cache memory in the box (unless told otherwise)
@@ -83,7 +83,7 @@ $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
if ($aCMDResult['create-db'] || $aCMDResult['all']) { if ($aCMDResult['create-db'] || $aCMDResult['all']) {
echo "Create DB\n"; info("Create DB");
$bDidSomething = true; $bDidSomething = true;
$oDB = DB::connect(CONST_Database_DSN, false); $oDB = DB::connect(CONST_Database_DSN, false);
if (!PEAR::isError($oDB)) { if (!PEAR::isError($oDB)) {
@@ -93,11 +93,9 @@ if ($aCMDResult['create-db'] || $aCMDResult['all']) {
} }
if ($aCMDResult['setup-db'] || $aCMDResult['all']) { if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
echo "Setup DB\n"; info("Setup DB");
$bDidSomething = true; $bDidSomething = true;
// TODO: path detection, detection memory, etc.
//
$oDB =& getDB(); $oDB =& getDB();
$fPostgresVersion = getPostgresVersion($oDB); $fPostgresVersion = getPostgresVersion($oDB);
@@ -117,7 +115,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
if ($iNumFunc == 0) { if ($iNumFunc == 0) {
pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable");
echo "WARNING: Postgresql is too old. extratags and namedetails API not available."; warn('Postgresql is too old. extratags and namedetails API not available.');
} }
$fPostgisVersion = getPostgisVersion($oDB); $fPostgisVersion = getPostgisVersion($oDB);
@@ -145,7 +143,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) {
pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz'); pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
} else { } else {
echo "WARNING: external UK postcode table not found.\n"; warn('external UK postcode table not found.');
} }
if (CONST_Use_Extra_US_Postcodes) { if (CONST_Use_Extra_US_Postcodes) {
pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
@@ -164,7 +162,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
} }
if ($aCMDResult['import-data'] || $aCMDResult['all']) { if ($aCMDResult['import-data'] || $aCMDResult['all']) {
echo "Import\n"; info('Import data');
$bDidSomething = true; $bDidSomething = true;
$osm2pgsql = CONST_Osm2pgsql_Binary; $osm2pgsql = CONST_Osm2pgsql_Binary;
@@ -198,16 +196,18 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) {
} }
if ($aCMDResult['create-functions'] || $aCMDResult['all']) { if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
echo "Functions\n"; info('Create Functions');
$bDidSomething = true; $bDidSomething = true;
if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built"); if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) {
fail("nominatim module not built");
}
create_sql_functions($aCMDResult); create_sql_functions($aCMDResult);
} }
if ($aCMDResult['create-tables'] || $aCMDResult['all']) { if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
info('Create Tables');
$bDidSomething = true; $bDidSomething = true;
echo "Tables\n";
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = replace_tablespace( $sTemplate = replace_tablespace(
@@ -243,12 +243,12 @@ if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
pgsqlRunScript($sTemplate, false); pgsqlRunScript($sTemplate, false);
// re-run the functions // re-run the functions
echo "Functions\n"; info('Recreate Functions');
create_sql_functions($aCMDResult); create_sql_functions($aCMDResult);
} }
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
echo "Partition Tables\n"; info('Create Partition Tables');
$bDidSomething = true; $bDidSomething = true;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
@@ -288,7 +288,7 @@ if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
echo "Partition Functions\n"; info('Create Partition Functions');
$bDidSomething = true; $bDidSomething = true;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
@@ -301,24 +301,22 @@ if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
$sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
$sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
if (file_exists($sWikiArticlesFile)) { if (file_exists($sWikiArticlesFile)) {
echo "Importing wikipedia articles..."; info('Importing wikipedia articles');
pgsqlRunDropAndRestore($sWikiArticlesFile); pgsqlRunDropAndRestore($sWikiArticlesFile);
echo "...done\n";
} else { } else {
echo "WARNING: wikipedia article dump file not found - places will have default importance\n"; warn('wikipedia article dump file not found - places will have default importance');
} }
if (file_exists($sWikiRedirectsFile)) { if (file_exists($sWikiRedirectsFile)) {
echo "Importing wikipedia redirects..."; info('Importing wikipedia redirects');
pgsqlRunDropAndRestore($sWikiRedirectsFile); pgsqlRunDropAndRestore($sWikiRedirectsFile);
echo "...done\n";
} else { } else {
echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n"; warn('wikipedia redirect dump file not found - some place importance values may be missing');
} }
} }
if ($aCMDResult['load-data'] || $aCMDResult['all']) { if ($aCMDResult['load-data'] || $aCMDResult['all']) {
echo "Drop old Data\n"; info('Drop old Data');
$bDidSomething = true; $bDidSomething = true;
$oDB =& getDB(); $oDB =& getDB();
@@ -361,11 +359,11 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
// pre-create the word list // pre-create the word list
if (!$aCMDResult['disable-token-precalc']) { if (!$aCMDResult['disable-token-precalc']) {
echo "Loading word list\n"; info('Loading word list');
pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
} }
echo "Load Data\n"; info('Load Data');
$sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry';
$aDBInstances = array(); $aDBInstances = array();
@@ -402,13 +400,13 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
echo '.'; echo '.';
} }
echo "\n"; echo "\n";
echo "Reanalysing database...\n"; info('Reanalysing database');
pgsqlRunScript('ANALYSE'); pgsqlRunScript('ANALYSE');
$sDatabaseDate = getDatabaseDate($oDB); $sDatabaseDate = getDatabaseDate($oDB);
pg_query($oDB->connection, 'TRUNCATE import_status'); pg_query($oDB->connection, 'TRUNCATE import_status');
if ($sDatabaseDate === false) { if ($sDatabaseDate === false) {
echo "WARNING: could not determine database date.\n"; warn('could not determine database date.');
} else { } else {
$sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
pg_query($oDB->connection, $sSQL); pg_query($oDB->connection, $sSQL);
@@ -417,6 +415,7 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
} }
if ($aCMDResult['import-tiger-data']) { if ($aCMDResult['import-tiger-data']) {
info('Import Tiger data');
$bDidSomething = true; $bDidSomething = true;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
@@ -474,7 +473,7 @@ if ($aCMDResult['import-tiger-data']) {
echo "\n"; echo "\n";
} }
echo "Creating indexes\n"; info('Creating indexes on Tiger data');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = replace_tablespace( $sTemplate = replace_tablespace(
@@ -491,6 +490,7 @@ if ($aCMDResult['import-tiger-data']) {
} }
if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) { if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
info('Calculate Postcodes');
$bDidSomething = true; $bDidSomething = true;
$oDB =& getDB(); $oDB =& getDB();
if (!pg_query($oDB->connection, 'TRUNCATE location_postcode')) { if (!pg_query($oDB->connection, 'TRUNCATE location_postcode')) {
@@ -557,20 +557,23 @@ if ($aCMDResult['index'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$sOutputFile = ''; $sOutputFile = '';
$sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile; $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
info('Index ranks 0 - 4');
passthruCheckReturn($sBaseCmd.' -R 4'); passthruCheckReturn($sBaseCmd.' -R 4');
if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE'); if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
info('Index ranks 5 - 25');
passthruCheckReturn($sBaseCmd.' -r 5 -R 25'); passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE'); if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
info('Index ranks 26 - 30');
passthruCheckReturn($sBaseCmd.' -r 26'); passthruCheckReturn($sBaseCmd.' -r 26');
echo "Indexing postcodes....\n"; info('Index postcodes');
$oDB =& getDB(); $oDB =& getDB();
$sSQL = 'UPDATE location_postcode SET indexed_status = 0'; $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
} }
if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
echo "Search indices\n"; info('Create Search indices');
$bDidSomething = true; $bDidSomething = true;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
@@ -594,7 +597,7 @@ if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
} }
if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
echo 'Creating search index for default country names'; info('Create search index for default country names');
$bDidSomething = true; $bDidSomething = true;
pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')");
@@ -620,6 +623,7 @@ if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
} }
if ($aCMDResult['drop']) { if ($aCMDResult['drop']) {
info('Drop tables only required for updates');
// The implementation is potentially a bit dangerous because it uses // The implementation is potentially a bit dangerous because it uses
// a positive selection of tables to keep, and deletes everything else. // a positive selection of tables to keep, and deletes everything else.
// Including any tables that the unsuspecting user might have manually // Including any tables that the unsuspecting user might have manually
@@ -676,7 +680,10 @@ if ($aCMDResult['drop']) {
if (!$bDidSomething) { if (!$bDidSomething) {
showUsage($aCMDOptions, true); showUsage($aCMDOptions, true);
} else { } else {
echo "Setup finished.\n"; echo "Summary of warnings:\n\n";
repeatWarnings();
echo "\n";
info('Setup finished.');
} }