mirror of
https://github.com/osm-search/Nominatim.git
synced 2026-03-13 14:24:08 +00:00
small fixes on setup.php and a bring update.php to work
This commit is contained in:
ThomasBarris
@@ -111,7 +111,6 @@ set(CUSTOMFILES
|
||||
utils/query.php
|
||||
utils/server_compare.php
|
||||
utils/setup.php
|
||||
utils/setupClass.php
|
||||
utils/specialphrases.php
|
||||
utils/update.php
|
||||
utils/warm.php
|
||||
|
||||
1794
utils/setupClass.php → lib/SetupClass.php
Executable file → Normal file
1794
utils/setupClass.php → lib/SetupClass.php
Executable file → Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,80 +1,126 @@
|
||||
<?php
|
||||
|
||||
function checkInFile($sOSMFile)
|
||||
{
|
||||
if (!isset($sOSMFile)) {
|
||||
fail('missing --osm-file for data import');
|
||||
}
|
||||
|
||||
if (!file_exists($sOSMFile)) {
|
||||
fail('the path supplied to --osm-file does not exist');
|
||||
}
|
||||
|
||||
if (!is_readable($sOSMFile)) {
|
||||
fail('osm-file "'.$aCMDResult['osm-file'].'" not readable');
|
||||
}
|
||||
}
|
||||
|
||||
function checkModulePresence()
|
||||
{
|
||||
// Try accessing the C module, so we know early if something is wrong
|
||||
// and can simply error out.
|
||||
$sModulePath = CONST_Database_Module_Path;
|
||||
$sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '";
|
||||
$sSQL .= $sModulePath."/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT";
|
||||
$sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);';
|
||||
|
||||
$oDB =& getDB();
|
||||
$oResult = $oDB->query($sSQL);
|
||||
|
||||
$bResult = true;
|
||||
|
||||
if (PEAR::isError($oResult)) {
|
||||
echo "\nERROR: Failed to load nominatim module. Reason:\n";
|
||||
echo $oResult->userinfo."\n\n";
|
||||
$bResult = false;
|
||||
}
|
||||
|
||||
return $bResult;
|
||||
}
|
||||
|
||||
function createSetupArgvArray()
|
||||
{
|
||||
$aCMDOptions
|
||||
= array(
|
||||
'Create and setup nominatim search system',
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
|
||||
array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
|
||||
array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
|
||||
|
||||
array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
|
||||
|
||||
array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
|
||||
array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
|
||||
array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
|
||||
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
|
||||
array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
|
||||
array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
|
||||
array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
|
||||
array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
|
||||
array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
|
||||
array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
|
||||
array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
|
||||
array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
|
||||
array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
|
||||
array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
|
||||
array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
|
||||
array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
|
||||
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
|
||||
array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
|
||||
array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
|
||||
array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
|
||||
array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
|
||||
array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'),
|
||||
array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'),
|
||||
);
|
||||
return $aCMDOptions;
|
||||
}
|
||||
<?php
|
||||
|
||||
function checkInFile($sOSMFile)
|
||||
{
|
||||
if (!isset($sOSMFile)) {
|
||||
fail('missing --osm-file for data import');
|
||||
}
|
||||
|
||||
if (!file_exists($sOSMFile)) {
|
||||
fail('the path supplied to --osm-file does not exist');
|
||||
}
|
||||
|
||||
if (!is_readable($sOSMFile)) {
|
||||
fail('osm-file "' . $aCMDResult['osm-file'] . '" not readable');
|
||||
}
|
||||
}
|
||||
|
||||
function checkModulePresence()
|
||||
{
|
||||
// Try accessing the C module, so we know early if something is wrong
|
||||
// and can simply error out.
|
||||
$sModulePath = CONST_Database_Module_Path;
|
||||
$sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '";
|
||||
$sSQL .= $sModulePath . "/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT";
|
||||
$sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);';
|
||||
|
||||
$oDB = &getDB();
|
||||
$oResult = $oDB->query($sSQL);
|
||||
|
||||
$bResult = true;
|
||||
|
||||
if (PEAR::isError($oResult)) {
|
||||
echo "\nERROR: Failed to load nominatim module. Reason:\n";
|
||||
echo $oResult->userinfo . "\n\n";
|
||||
$bResult = false;
|
||||
}
|
||||
|
||||
return $bResult;
|
||||
}
|
||||
|
||||
// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
|
||||
// create and array
|
||||
function createSetupArgvArray()
|
||||
{
|
||||
$aCMDOptions
|
||||
= array(
|
||||
'Create and setup nominatim search system',
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
|
||||
array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
|
||||
array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
|
||||
|
||||
array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
|
||||
|
||||
array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
|
||||
array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
|
||||
array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
|
||||
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
|
||||
array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
|
||||
array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
|
||||
array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
|
||||
array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
|
||||
array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
|
||||
array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
|
||||
array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
|
||||
array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
|
||||
array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
|
||||
array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
|
||||
array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
|
||||
array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
|
||||
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
|
||||
array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
|
||||
array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
|
||||
array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
|
||||
array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
|
||||
array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'),
|
||||
array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'),
|
||||
);
|
||||
return $aCMDOptions;
|
||||
}
|
||||
|
||||
function createUpdateArgvArray()
|
||||
{
|
||||
$aCMDOptions
|
||||
= array(
|
||||
'Import / update / index osm data',
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
|
||||
array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
|
||||
array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
|
||||
array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
|
||||
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
|
||||
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
|
||||
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
|
||||
|
||||
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
|
||||
|
||||
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
|
||||
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
|
||||
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
|
||||
|
||||
array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
|
||||
array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
|
||||
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
|
||||
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
|
||||
|
||||
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
|
||||
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
|
||||
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
|
||||
|
||||
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
|
||||
array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
|
||||
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
|
||||
|
||||
array('create-functions', '', 0, 1, 1, 1, 'bool', 'Create functions'),
|
||||
array('enable-diff-updates', '', 0, 1, 1, 1, 'bool', 'Turn on the code required to make diff updates work'),
|
||||
array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
|
||||
array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
|
||||
array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
|
||||
);
|
||||
return $aCMDOptions;
|
||||
}
|
||||
|
||||
266
utils/setup.php
266
utils/setup.php
@@ -1,133 +1,133 @@
|
||||
#!@PHP_BIN@ -Cq
|
||||
<?php
|
||||
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
include_once(CONST_InstallPath.'/utils/setupClass.php');
|
||||
// ->indirect via init-cmd.php -> /lib/cmd.php for runWithEnv, getCmdOpt
|
||||
// ->indirect via init-cmd.php -> /lib/init.php -> db.php for &getDB()
|
||||
|
||||
include_once(CONST_BasePath.'/lib/setup_functions.php');
|
||||
include_once(CONST_BasePath.'/lib/setup_functions.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
|
||||
|
||||
$aCMDOptions = createSetupArgvArray();
|
||||
|
||||
// $aCMDOptions passed to getCmdOpt by reference
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
|
||||
|
||||
|
||||
$bDidSomething = false;
|
||||
|
||||
//*******************************************************
|
||||
// Making some sanity check:
|
||||
// Check if osm-file is set and points to a valid file
|
||||
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
|
||||
// to remain in /lib/setup_functions.php function
|
||||
checkInFile($aCMDResult['osm-file']);
|
||||
echo $aCMDResult['osm-file'];
|
||||
}
|
||||
|
||||
// osmosis init is no longer supported
|
||||
if ($aCMDResult['osmosis-init']) {
|
||||
$bDidSomething = true;
|
||||
echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n";
|
||||
}
|
||||
|
||||
// ******************************************************
|
||||
// instantiate Setup class
|
||||
$cSetup = new SetupFunctions($aCMDResult);
|
||||
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> createDB();
|
||||
}
|
||||
|
||||
// *******************************************************
|
||||
// go through complete process if 'all' is selected or start selected functions
|
||||
if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> setupDB();
|
||||
}
|
||||
|
||||
// Try accessing the C module, so we know early if something is wrong
|
||||
if (!checkModulePresence()) {
|
||||
fail('error loading nominatim.so module');
|
||||
}
|
||||
|
||||
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> importData($aCMDResult['osm-file']);
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> createFunctions();
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> createTables();
|
||||
$cSetup -> recreateFunction();
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> createPartitionTables();
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> createPartitionFunctions();
|
||||
}
|
||||
|
||||
if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> importWikipediaArticles();
|
||||
}
|
||||
|
||||
if ($aCMDResult['load-data'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> loadData($aCMDResult['disable-token-precalc']);
|
||||
}
|
||||
|
||||
if ($aCMDResult['import-tiger-data']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> importTigerData();
|
||||
}
|
||||
|
||||
if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> calculatePostcodes($aCMDResult['all']);
|
||||
}
|
||||
|
||||
if ($aCMDResult['index'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> index($aCMDResult['index-noanalyse']);
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> createSearchIndices();
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> createCountryNames($aCMDResult);
|
||||
}
|
||||
|
||||
if ($aCMDResult['drop']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup -> drop($aCMDResult);
|
||||
}
|
||||
|
||||
// ******************************************************
|
||||
// If we did something, repeat the warnings
|
||||
if (!$bDidSomething) {
|
||||
showUsage($aCMDOptions, true);
|
||||
} else {
|
||||
echo "Summary of warnings:\n\n";
|
||||
repeatWarnings();
|
||||
echo "\n";
|
||||
info('Setup finished.');
|
||||
}
|
||||
#!@PHP_BIN@ -Cq
|
||||
<?php
|
||||
|
||||
require_once dirname(dirname(__FILE__)) . '/settings/settings.php';
|
||||
require_once CONST_BasePath . '/lib/init-cmd.php';
|
||||
require_once CONST_BasePath . '/lib/SetupClass.php';
|
||||
// ->indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt
|
||||
// ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB()
|
||||
|
||||
require_once CONST_BasePath . '/lib/setup_functions.php';
|
||||
ini_set('memory_limit', '800M');
|
||||
|
||||
use Nominatim\Setup\SetupFunctions as SetupFunctions;
|
||||
|
||||
$aCMDOptions = createSetupArgvArray();
|
||||
|
||||
// $aCMDOptions passed to getCmdOpt by reference
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
|
||||
|
||||
$bDidSomething = false;
|
||||
|
||||
//*******************************************************
|
||||
// Making some sanity check:
|
||||
// Check if osm-file is set and points to a valid file
|
||||
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
|
||||
// to remain in /lib/setup_functions.php function
|
||||
checkInFile($aCMDResult['osm-file']);
|
||||
echo $aCMDResult['osm-file'];
|
||||
}
|
||||
|
||||
// osmosis init is no longer supported
|
||||
if ($aCMDResult['osmosis-init']) {
|
||||
$bDidSomething = true;
|
||||
echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n";
|
||||
}
|
||||
|
||||
// ******************************************************
|
||||
// instantiate Setup class
|
||||
$cSetup = new SetupFunctions($aCMDResult);
|
||||
|
||||
// *******************************************************
|
||||
// go through complete process if 'all' is selected or start selected functions
|
||||
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->createDB();
|
||||
}
|
||||
|
||||
if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->setupDB();
|
||||
}
|
||||
|
||||
// Try accessing the C module, so we know early if something is wrong
|
||||
if (!checkModulePresence()) {
|
||||
fail('error loading nominatim.so module');
|
||||
}
|
||||
|
||||
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->importData($aCMDResult['osm-file']);
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->createFunctions();
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->createTables();
|
||||
$cSetup->createFunctions();
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->createPartitionTables();
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->createPartitionFunctions();
|
||||
}
|
||||
|
||||
if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->importWikipediaArticles();
|
||||
}
|
||||
|
||||
if ($aCMDResult['load-data'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->loadData($aCMDResult['disable-token-precalc']);
|
||||
}
|
||||
|
||||
if ($aCMDResult['import-tiger-data']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->importTigerData();
|
||||
}
|
||||
|
||||
if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->calculatePostcodes($aCMDResult['all']);
|
||||
}
|
||||
|
||||
if ($aCMDResult['index'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->index($aCMDResult['index-noanalyse']);
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->createSearchIndices();
|
||||
}
|
||||
|
||||
if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->createCountryNames($aCMDResult);
|
||||
}
|
||||
|
||||
if ($aCMDResult['drop']) {
|
||||
$bDidSomething = true;
|
||||
$cSetup->drop($aCMDResult);
|
||||
}
|
||||
|
||||
// ******************************************************
|
||||
// If we did something, repeat the warnings
|
||||
if (!$bDidSomething) {
|
||||
showUsage($aCMDOptions, true);
|
||||
} else {
|
||||
echo "Summary of warnings:\n\n";
|
||||
repeatWarnings();
|
||||
echo "\n";
|
||||
info('Setup finished.');
|
||||
}
|
||||
|
||||
190
utils/update.php
190
utils/update.php
@@ -1,66 +1,42 @@
|
||||
#!@PHP_BIN@ -Cq
|
||||
<?php
|
||||
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
include_once(CONST_BasePath.'/lib/setup_functions.php');
|
||||
require_once dirname(dirname(__FILE__)) . '/settings/settings.php';
|
||||
require_once CONST_BasePath . '/lib/init-cmd.php';
|
||||
require_once CONST_BasePath . '/lib/setup_functions.php';
|
||||
require_once CONST_BasePath . '/lib/SetupClass.php';
|
||||
ini_set('memory_limit', '800M');
|
||||
|
||||
use Nominatim\Setup\SetupFunctions as SetupFunctions;
|
||||
|
||||
# (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
|
||||
$aCMDOptions = createUpdateArgvArray();
|
||||
|
||||
$aCMDOptions
|
||||
= array(
|
||||
'Import / update / index osm data',
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
|
||||
array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
|
||||
array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
|
||||
array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
|
||||
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
|
||||
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
|
||||
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
|
||||
|
||||
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
|
||||
|
||||
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
|
||||
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
|
||||
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
|
||||
|
||||
array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
|
||||
array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
|
||||
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
|
||||
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
|
||||
|
||||
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
|
||||
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
|
||||
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
|
||||
|
||||
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
|
||||
array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
|
||||
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
|
||||
|
||||
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
|
||||
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
|
||||
if (!isset($aResult['index-instances'])) {
|
||||
$aResult['index-instances'] = 1;
|
||||
}
|
||||
|
||||
if (!isset($aResult['index-rank'])) {
|
||||
$aResult['index-rank'] = 0;
|
||||
}
|
||||
|
||||
date_default_timezone_set('Etc/UTC');
|
||||
|
||||
$oDB =& getDB();
|
||||
$oDB = &getDB();
|
||||
|
||||
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
|
||||
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
|
||||
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) {
|
||||
$aDSNInfo['port'] = 5432;
|
||||
}
|
||||
|
||||
// cache memory to be used by osm2pgsql, should not be more than the available memory
|
||||
$iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
|
||||
$iCacheMemory = (isset($aResult['osm2pgsql-cache']) ? $aResult['osm2pgsql-cache'] : 2000);
|
||||
if ($iCacheMemory + 500 > getTotalMemoryMB()) {
|
||||
$iCacheMemory = getCacheMemoryMB();
|
||||
echo "WARNING: resetting cache memory to $iCacheMemory\n";
|
||||
}
|
||||
$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
|
||||
$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary . ' -klas --number-processes 1 -C ' . $iCacheMemory . ' -O gazetteer -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'];
|
||||
if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
|
||||
$sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username'];
|
||||
}
|
||||
@@ -73,12 +49,12 @@ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
|
||||
}
|
||||
|
||||
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
|
||||
$sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
|
||||
$sOsm2pgsqlCmd .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File;
|
||||
}
|
||||
|
||||
if ($aResult['init-updates']) {
|
||||
// sanity check that the replication URL is correct
|
||||
$sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt');
|
||||
$sBaseState = file_get_contents(CONST_Replication_Url . '/state.txt');
|
||||
if ($sBaseState === false) {
|
||||
echo "\nCannot find state.txt file at the configured replication URL.\n";
|
||||
echo "Does the URL point to a directory containing OSM update data?\n\n";
|
||||
@@ -92,7 +68,7 @@ if ($aResult['init-updates']) {
|
||||
fail('CONST_Pyosmium_Binary not configured');
|
||||
}
|
||||
$aOutput = 0;
|
||||
$sCmd = CONST_Pyosmium_Binary.' --help';
|
||||
$sCmd = CONST_Pyosmium_Binary . ' --help';
|
||||
exec($sCmd, $aOutput, $iRet);
|
||||
if ($iRet != 0) {
|
||||
echo "Cannot execute pyosmium-get-changes.\n";
|
||||
@@ -100,19 +76,23 @@ if ($aResult['init-updates']) {
|
||||
echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n";
|
||||
fail('pyosmium-get-changes not found or not usable');
|
||||
}
|
||||
|
||||
if (!$aResult['no-update-functions']) {
|
||||
createFunctions($aCMDResult);
|
||||
// instatiate setupClass to use the function therein
|
||||
// instantiate Setup class
|
||||
$cSetup = new SetupFunctions($aResult);
|
||||
$cSetup->createFunctions();
|
||||
}
|
||||
|
||||
$sDatabaseDate = getDatabaseDate($oDB);
|
||||
if ($sDatabaseDate === false) {
|
||||
fail('Cannot determine date of database.');
|
||||
}
|
||||
$sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60));
|
||||
$sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3 * 60 * 60));
|
||||
|
||||
// get the appropriate state id
|
||||
$aOutput = 0;
|
||||
$sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url;
|
||||
$sCmd = CONST_Pyosmium_Binary . ' -D ' . $sWindBack . ' --server ' . CONST_Replication_Url;
|
||||
exec($sCmd, $aOutput, $iRet);
|
||||
if ($iRet != 0 || $aOutput[0] == 'None') {
|
||||
fail('Error running pyosmium tools');
|
||||
@@ -120,7 +100,7 @@ if ($aResult['init-updates']) {
|
||||
|
||||
pg_query($oDB->connection, 'TRUNCATE import_status');
|
||||
$sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('";
|
||||
$sSQL .= $sDatabaseDate."',".$aOutput[0].', true)';
|
||||
$sSQL .= $sDatabaseDate . "'," . $aOutput[0] . ', true)';
|
||||
if (!pg_query($oDB->connection, $sSQL)) {
|
||||
fail('Could not enter sequence into database.');
|
||||
}
|
||||
@@ -135,7 +115,7 @@ if ($aResult['check-for-updates']) {
|
||||
fail('Updates not set up. Please run ./utils/update.php --init-updates.');
|
||||
}
|
||||
|
||||
system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet);
|
||||
system(CONST_BasePath . '/utils/check_server_for_updates.py ' . CONST_Replication_Url . ' ' . $aLastState['sequence_id'], $iRet);
|
||||
exit($iRet);
|
||||
}
|
||||
|
||||
@@ -148,8 +128,8 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
|
||||
}
|
||||
|
||||
// Import the file
|
||||
$sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
|
||||
echo $sCMD."\n";
|
||||
$sCMD = $sOsm2pgsqlCmd . ' ' . $sNextFile;
|
||||
echo $sCMD . "\n";
|
||||
$iErrorLevel = runWithEnv($sCMD, $aProcEnv);
|
||||
|
||||
if ($iErrorLevel) {
|
||||
@@ -161,35 +141,35 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
|
||||
|
||||
if ($aResult['calculate-postcodes']) {
|
||||
info('Update postcodes centroids');
|
||||
$sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql');
|
||||
$sTemplate = file_get_contents(CONST_BasePath . '/sql/update-postcodes.sql');
|
||||
runSQLScript($sTemplate, true, true);
|
||||
}
|
||||
|
||||
$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
|
||||
$sTemporaryFile = CONST_BasePath . '/data/osmosischange.osc';
|
||||
$bHaveDiff = false;
|
||||
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
|
||||
$sContentURL = '';
|
||||
if (isset($aResult['import-node']) && $aResult['import-node']) {
|
||||
if ($bUseOSMApi) {
|
||||
$sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
|
||||
$sContentURL = 'https://www.openstreetmap.org/api/0.6/node/' . $aResult['import-node'];
|
||||
} else {
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=node(' . $aResult['import-node'] . ');out%20meta;';
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($aResult['import-way']) && $aResult['import-way']) {
|
||||
if ($bUseOSMApi) {
|
||||
$sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
|
||||
$sContentURL = 'https://www.openstreetmap.org/api/0.6/way/' . $aResult['import-way'] . '/full';
|
||||
} else {
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=(way(' . $aResult['import-way'] . ');node(w););out%20meta;';
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($aResult['import-relation']) && $aResult['import-relation']) {
|
||||
if ($bUseOSMApi) {
|
||||
$sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
|
||||
$sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/' . $aResult['import-relation'] . '/full';
|
||||
} else {
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
|
||||
$sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel(' . $aResult['import-relation'] . ');way(r);node(w));node(r));out%20meta;';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,8 +180,8 @@ if ($sContentURL) {
|
||||
|
||||
if ($bHaveDiff) {
|
||||
// import generated change file
|
||||
$sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
|
||||
echo $sCMD."\n";
|
||||
$sCMD = $sOsm2pgsqlCmd . ' ' . $sTemporaryFile;
|
||||
echo $sCMD . "\n";
|
||||
$iErrorLevel = runWithEnv($sCMD, $aProcEnv);
|
||||
if ($iErrorLevel) {
|
||||
fail("osm2pgsql exited with error level $iErrorLevel\n");
|
||||
@@ -209,7 +189,7 @@ if ($bHaveDiff) {
|
||||
}
|
||||
|
||||
if ($aResult['deduplicate']) {
|
||||
$oDB =& getDB();
|
||||
$oDB = &getDB();
|
||||
|
||||
if (getPostgresVersion($oDB) < 9.3) {
|
||||
fail('ERROR: deduplicate is only currently supported in postgresql 9.3');
|
||||
@@ -221,7 +201,7 @@ if ($aResult['deduplicate']) {
|
||||
|
||||
// we don't care about empty search_name_* partitions, they can't contain mentions of duplicates
|
||||
foreach ($aPartitions as $i => $sPartition) {
|
||||
$sSQL = 'select count(*) from search_name_'.$sPartition;
|
||||
$sSQL = 'select count(*) from search_name_' . $sPartition;
|
||||
$nEntries = chksql($oDB->getOne($sSQL));
|
||||
if ($nEntries == 0) {
|
||||
unset($aPartitions[$i]);
|
||||
@@ -233,11 +213,14 @@ if ($aResult['deduplicate']) {
|
||||
$sSQL .= ' group by word_token having count(*) > 1 order by word_token';
|
||||
$aDuplicateTokens = chksql($oDB->getAll($sSQL));
|
||||
foreach ($aDuplicateTokens as $aToken) {
|
||||
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
|
||||
echo 'Deduping '.$aToken['word_token']."\n";
|
||||
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') {
|
||||
continue;
|
||||
}
|
||||
|
||||
echo 'Deduping ' . $aToken['word_token'] . "\n";
|
||||
$sSQL = 'select word_id,';
|
||||
$sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num';
|
||||
$sSQL .= " from word where word_token = '".$aToken['word_token'];
|
||||
$sSQL .= " from word where word_token = '" . $aToken['word_token'];
|
||||
$sSQL .= "' and class is null and type is null and country_code is null order by num desc";
|
||||
$aTokenSet = chksql($oDB->getAll($sSQL));
|
||||
|
||||
@@ -246,34 +229,34 @@ if ($aResult['deduplicate']) {
|
||||
|
||||
foreach ($aTokenSet as $aRemove) {
|
||||
$sSQL = 'update search_name set';
|
||||
$sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),';
|
||||
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
|
||||
$sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
|
||||
$sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . '),';
|
||||
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
|
||||
$sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']';
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
$sSQL = 'update search_name set';
|
||||
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
|
||||
$sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']';
|
||||
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
|
||||
$sSQL .= ' where nameaddress_vector @> ARRAY[' . $aRemove['word_id'] . ']';
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
$sSQL = 'update location_area_country set';
|
||||
$sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
|
||||
$sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
|
||||
$sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
|
||||
$sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']';
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
foreach ($aPartitions as $sPartition) {
|
||||
$sSQL = 'update search_name_'.$sPartition.' set';
|
||||
$sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')';
|
||||
$sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
|
||||
$sSQL = 'update search_name_' . $sPartition . ' set';
|
||||
$sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
|
||||
$sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']';
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
$sSQL = 'update location_area_country set';
|
||||
$sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
|
||||
$sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
|
||||
$sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
|
||||
$sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']';
|
||||
chksql($oDB->query($sSQL));
|
||||
}
|
||||
|
||||
$sSQL = 'delete from word where word_id = '.$aRemove['word_id'];
|
||||
$sSQL = 'delete from word where word_id = ' . $aRemove['word_id'];
|
||||
chksql($oDB->query($sSQL));
|
||||
}
|
||||
}
|
||||
@@ -281,12 +264,12 @@ if ($aResult['deduplicate']) {
|
||||
|
||||
if ($aResult['recompute-word-counts']) {
|
||||
info('Recompute frequency of full-word search terms');
|
||||
$sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql');
|
||||
$sTemplate = file_get_contents(CONST_BasePath . '/sql/words_from_search_name.sql');
|
||||
runSQLScript($sTemplate, true, true);
|
||||
}
|
||||
|
||||
if ($aResult['index']) {
|
||||
$sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank'];
|
||||
$sCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'] . ' -r ' . $aResult['index-rank'];
|
||||
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
|
||||
$sCmd .= ' -H ' . $aDSNInfo['hostspec'];
|
||||
}
|
||||
@@ -301,13 +284,13 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
//
|
||||
if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
|
||||
fail('Error: Update interval too low for download.geofabrik.de. ' .
|
||||
"Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");
|
||||
"Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");
|
||||
}
|
||||
|
||||
$sImportFile = CONST_InstallPath.'/osmosischange.osc';
|
||||
$sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;
|
||||
$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
|
||||
$sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
|
||||
$sImportFile = CONST_InstallPath . '/osmosischange.osc';
|
||||
$sCMDDownload = CONST_Pyosmium_Binary . ' --server ' . CONST_Replication_Url . ' -o ' . $sImportFile . ' -s ' . CONST_Replication_Max_Diff_size;
|
||||
$sCMDImport = $sOsm2pgsqlCmd . ' ' . $sImportFile;
|
||||
$sCMDIndex = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'];
|
||||
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
|
||||
$sCMDIndex .= ' -H ' . $aDSNInfo['hostspec'];
|
||||
}
|
||||
@@ -324,7 +307,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n";
|
||||
echo 'Currently at sequence ' . $aLastState['sequence_id'] . ' (' . $aLastState['lastimportdate'] . ') - ' . $aLastState['indexed'] . " indexed\n";
|
||||
|
||||
$sBatchEnd = $aLastState['lastimportdate'];
|
||||
$iEndSequence = $aLastState['sequence_id'];
|
||||
@@ -347,23 +330,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
if (file_exists($sImportFile)) {
|
||||
unlink($sImportFile);
|
||||
}
|
||||
exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
|
||||
exec($sCMDDownload . ' -I ' . $iNextSeq, $aOutput, $iResult);
|
||||
|
||||
if ($iResult == 3) {
|
||||
echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
|
||||
echo 'No new updates. Sleeping for ' . CONST_Replication_Recheck_Interval . " sec.\n";
|
||||
sleep(CONST_Replication_Recheck_Interval);
|
||||
} elseif ($iResult != 0) {
|
||||
echo 'ERROR: updates failed.';
|
||||
exit($iResult);
|
||||
} else {
|
||||
$iEndSequence = (int)$aOutput[0];
|
||||
$iEndSequence = (int) $aOutput[0];
|
||||
}
|
||||
} while ($iResult);
|
||||
|
||||
// get the newest object from the diff file
|
||||
$sBatchEnd = 0;
|
||||
$iRet = 0;
|
||||
exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
|
||||
exec(CONST_BasePath . '/utils/osm_file_date.py ' . $sImportFile, $sBatchEnd, $iRet);
|
||||
if ($iRet == 5) {
|
||||
echo "Diff file is empty. skipping import.\n";
|
||||
if (!$aResult['import-osmosis-all']) {
|
||||
@@ -379,7 +362,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
|
||||
// Import the file
|
||||
$fCMDStartTime = time();
|
||||
echo $sCMDImport."\n";
|
||||
echo $sCMDImport . "\n";
|
||||
unset($sJunk);
|
||||
$iErrorLevel = runWithEnv($sCMDImport, $aProcEnv);
|
||||
if ($iErrorLevel) {
|
||||
@@ -392,8 +375,8 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
$sSQL = 'INSERT INTO import_osmosis_log';
|
||||
$sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
|
||||
$sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
|
||||
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
|
||||
$sSQL .= date('Y-m-d H:i:s')."','import')";
|
||||
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','";
|
||||
$sSQL .= date('Y-m-d H:i:s') . "','import')";
|
||||
var_Dump($sSQL);
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
@@ -401,7 +384,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
$sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence";
|
||||
var_Dump($sSQL);
|
||||
chksql($oDB->query($sSQL));
|
||||
echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
|
||||
echo date('Y-m-d H:i:s') . " Completed download step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n";
|
||||
}
|
||||
|
||||
// Index file
|
||||
@@ -419,18 +402,21 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
|
||||
$sSQL = 'INSERT INTO import_osmosis_log';
|
||||
$sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
|
||||
$sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
|
||||
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
|
||||
$sSQL .= date('Y-m-d H:i:s')."','index')";
|
||||
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','";
|
||||
$sSQL .= date('Y-m-d H:i:s') . "','index')";
|
||||
var_Dump($sSQL);
|
||||
$oDB->query($sSQL);
|
||||
echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
|
||||
echo date('Y-m-d H:i:s') . " Completed index step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n";
|
||||
|
||||
$sSQL = 'update import_status set indexed = true';
|
||||
$oDB->query($sSQL);
|
||||
}
|
||||
|
||||
$fDuration = time() - $fStartTime;
|
||||
echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n";
|
||||
if (!$aResult['import-osmosis-all']) exit(0);
|
||||
echo date('Y-m-d H:i:s') . " Completed all for $sBatchEnd in " . round($fDuration / 60, 2) . " minutes\n";
|
||||
if (!$aResult['import-osmosis-all']) {
|
||||
exit(0);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user