hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-16 15:47:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

small fixes on setup.php and a bring update.php to work

Browse Source
This commit is contained in:
ThomasBarris
2018-08-31 21:31:38 +02:00
parent b2f3cfde0b
commit a3b4f80c99
5 changed files with 1297 additions and 1160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

266
utils/setup.php
View File

@@ -1,133 +1,133 @@
#!@PHP_BIN@ -Cq
<?php
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
require_once(CONST_BasePath.'/lib/init-cmd.php');
include_once(CONST_InstallPath.'/utils/setupClass.php');
// ->indirect via init-cmd.php -> /lib/cmd.php for runWithEnv, getCmdOpt
// ->indirect via init-cmd.php -> /lib/init.php -> db.php for &getDB()
include_once(CONST_BasePath.'/lib/setup_functions.php');
include_once(CONST_BasePath.'/lib/setup_functions.php');
ini_set('memory_limit', '800M');
$aCMDOptions = createSetupArgvArray();
// $aCMDOptions passed to getCmdOpt by reference
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
$bDidSomething = false;
//*******************************************************
// Making some sanity check:
// Check if osm-file is set and points to a valid file
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
// to remain in /lib/setup_functions.php function
checkInFile($aCMDResult['osm-file']);
echo $aCMDResult['osm-file'];
}
// osmosis init is no longer supported
if ($aCMDResult['osmosis-init']) {
$bDidSomething = true;
echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n";
}
// ******************************************************
// instantiate Setup class
$cSetup = new SetupFunctions($aCMDResult);
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> createDB();
}
// *******************************************************
// go through complete process if 'all' is selected or start selected functions
if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> setupDB();
}
// Try accessing the C module, so we know early if something is wrong
if (!checkModulePresence()) {
fail('error loading nominatim.so module');
}
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> importData($aCMDResult['osm-file']);
}
if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> createFunctions();
}
if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> createTables();
$cSetup -> recreateFunction();
}
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> createPartitionTables();
}
if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> createPartitionFunctions();
}
if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> importWikipediaArticles();
}
if ($aCMDResult['load-data'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> loadData($aCMDResult['disable-token-precalc']);
}
if ($aCMDResult['import-tiger-data']) {
$bDidSomething = true;
$cSetup -> importTigerData();
}
if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> calculatePostcodes($aCMDResult['all']);
}
if ($aCMDResult['index'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> index($aCMDResult['index-noanalyse']);
}
if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> createSearchIndices();
}
if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> createCountryNames($aCMDResult);
}
if ($aCMDResult['drop']) {
$bDidSomething = true;
$cSetup -> drop($aCMDResult);
}
// ******************************************************
// If we did something, repeat the warnings
if (!$bDidSomething) {
showUsage($aCMDOptions, true);
} else {
echo "Summary of warnings:\n\n";
repeatWarnings();
echo "\n";
info('Setup finished.');
}
#!@PHP_BIN@ -Cq
<?php
require_once dirname(dirname(__FILE__)) . '/settings/settings.php';
require_once CONST_BasePath . '/lib/init-cmd.php';
require_once CONST_BasePath . '/lib/SetupClass.php';
// ->indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt
// ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB()
require_once CONST_BasePath . '/lib/setup_functions.php';
ini_set('memory_limit', '800M');
use Nominatim\Setup\SetupFunctions as SetupFunctions;
$aCMDOptions = createSetupArgvArray();
// $aCMDOptions passed to getCmdOpt by reference
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
$bDidSomething = false;
//*******************************************************
// Making some sanity check:
// Check if osm-file is set and points to a valid file
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
// to remain in /lib/setup_functions.php function
checkInFile($aCMDResult['osm-file']);
echo $aCMDResult['osm-file'];
}
// osmosis init is no longer supported
if ($aCMDResult['osmosis-init']) {
$bDidSomething = true;
echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n";
}
// ******************************************************
// instantiate Setup class
$cSetup = new SetupFunctions($aCMDResult);
// *******************************************************
// go through complete process if 'all' is selected or start selected functions
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->createDB();
}
if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->setupDB();
}
// Try accessing the C module, so we know early if something is wrong
if (!checkModulePresence()) {
fail('error loading nominatim.so module');
}
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->importData($aCMDResult['osm-file']);
}
if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->createFunctions();
}
if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->createTables();
$cSetup->createFunctions();
}
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->createPartitionTables();
}
if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->createPartitionFunctions();
}
if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->importWikipediaArticles();
}
if ($aCMDResult['load-data'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->loadData($aCMDResult['disable-token-precalc']);
}
if ($aCMDResult['import-tiger-data']) {
$bDidSomething = true;
$cSetup->importTigerData();
}
if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->calculatePostcodes($aCMDResult['all']);
}
if ($aCMDResult['index'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->index($aCMDResult['index-noanalyse']);
}
if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->createSearchIndices();
}
if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->createCountryNames($aCMDResult);
}
if ($aCMDResult['drop']) {
$bDidSomething = true;
$cSetup->drop($aCMDResult);
}
// ******************************************************
// If we did something, repeat the warnings
if (!$bDidSomething) {
showUsage($aCMDOptions, true);
} else {
echo "Summary of warnings:\n\n";
repeatWarnings();
echo "\n";
info('Setup finished.');
}

844
utils/setupClass.php
View File

@@ -1,844 +0,0 @@
<?php
class SetupFunctions
{
protected $iCacheMemory; // set in constructor
protected $iInstances; // set in constructor
protected $sModulePath; // set in constructor
protected $aDSNInfo; // set in constructor = DB::parseDSN(CONST_Database_DSN);
protected $sVerbose; // set in constructor
protected $sIgnoreErrors; // set in constructor
protected $bEnableDiffUpdates; // set in constructor
protected $bEnableDebugStatements; // set in constructor
protected $bNoPartitions; // set in constructor
protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData
// pgsqlRunPartitionScript, calculatePostcodes, ..if no already set
public function __construct($aCMDResult) {
// by default, use all but one processor, but never more than 15.
$this->iInstances = isset($aCMDResult['threads'])
? $aCMDResult['threads']
: (min(16, getProcessorCount()) - 1);
if ($this->iInstances < 1) {
$this->iInstances = 1;
warn('resetting threads to '.$this->iInstances);
}
// Assume we can steal all the cache memory in the box (unless told otherwise)
if (isset($aCMDResult['osm2pgsql-cache'])) {
$this->iCacheMemory = $aCMDResult['osm2pgsql-cache'];
} else {
$this->iCacheMemory = getCacheMemoryMB();
}
$this->sModulePath = CONST_Database_Module_Path;
info('module path: ' . $this->sModulePath);
// prepares DB for import or update, sets the Data Source Name
$this->aDSNInfo = DB::parseDSN(CONST_Database_DSN);
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
// setting member variables based on command line options stored in $aCMDResult
$this->sVerbose = $aCMDResult['verbose'];
$this->sIgnoreErrors = $aCMDResult['ignore-errors'];
$this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates'];
$this->bEnableDebugStatements = $aCMDResult['enable-debug-statements'];
$this->bNoPartitions = $aCMDResult['no-partitions'];
}
public function createDB()
{
info('Create DB');
$sDB = DB::connect(CONST_Database_DSN, false);
if (!PEAR::isError($sDB)) {
fail('database already exists ('.CONST_Database_DSN.')');
}
$sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database'];
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username'];
}
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sCreateDBCmd .= ' -h ' . $this->aDSNInfo['hostspec'];
}
$aProcEnv = null;
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
}
$result = runWithEnv($sCreateDBCmd, $aProcEnv);
if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd);
}
public function setupDB()
{
info('Setup DB');
$this->oDB =& getDB();
$fPostgresVersion = getPostgresVersion($this->oDB);
echo 'Postgres version found: '.$fPostgresVersion."\n";
if ($fPostgresVersion < 9.1) {
fail('Minimum supported version of Postgresql is 9.1.');
}
$this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
$this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
// For extratags and namedetails the hstore_to_json converter is
// needed which is only available from Postgresql 9.3+. For older
// versions add a dummy function that returns nothing.
$iNumFunc = chksql($this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'"));
if ($iNumFunc == 0) {
$this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable");
warn('Postgresql is too old. extratags and namedetails API not available.');
}
$fPostgisVersion = getPostgisVersion($this->oDB);
echo 'Postgis version found: '.$fPostgisVersion."\n";
if ($fPostgisVersion < 2.1) {
// Functions were renamed in 2.1 and throw an annoying deprecation warning
$this->pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
$this->pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint');
}
if ($fPostgisVersion < 2.2) {
$this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
}
$i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'"));
if ($i == 0) {
echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
echo "\n createuser ".CONST_Database_Web_User."\n\n";
exit(1);
}
if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) {
echo 'Error: you need to download the country_osm_grid first:';
echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n";
exit(1);
}
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) {
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
} else {
warn('external UK postcode table not found.');
}
if (CONST_Use_Extra_US_Postcodes) {
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
}
if ($this->bNoPartitions) {
$this->pgsqlRunScript('update country_name set partition = 0');
}
// the following will be needed by create_functions later but
// is only defined in the subsequently called T
// Create dummies here that will be overwritten by the proper
// versions in create-tables.
$this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()');
$this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false);
}
public function importData($sOSMFile)
{
info('Import data');
$osm2pgsql = CONST_Osm2pgsql_Binary;
if (!file_exists($osm2pgsql)) {
echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n";
echo "Normally you should not need to set this manually.\n";
fail("osm2pgsql not found in '$osm2pgsql'");
}
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
$osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
}
if (CONST_Tablespace_Osm2pgsql_Data)
$osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data;
if (CONST_Tablespace_Osm2pgsql_Index)
$osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index;
if (CONST_Tablespace_Place_Data)
$osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data;
if (CONST_Tablespace_Place_Index)
$osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index;
$osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
$osm2pgsql .= ' -C '.$this->iCacheMemory;
$osm2pgsql .= ' -P '.$this->aDSNInfo['port'];
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$osm2pgsql .= ' -U ' . $this->aDSNInfo['username'];
}
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$osm2pgsql .= ' -H ' . $this->aDSNInfo['hostspec'];
}
$aProcEnv = null;
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
}
$osm2pgsql .= ' -d '.$this->aDSNInfo['database'].' '.$sOSMFile;
runWithEnv($osm2pgsql, $aProcEnv);
if ($this->oDB == null) $this->oDB =& getDB();
if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) {
fail('No Data');
}
}
public function createFunctions()
{
info('Create Functions');
$this->createSqlFunctions();
}
public function createTables()
{
info('Create Tables');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:address-data}',
CONST_Tablespace_Address_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:address-index}',
CONST_Tablespace_Address_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:search-data}',
CONST_Tablespace_Search_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:search-index}',
CONST_Tablespace_Search_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-data}',
CONST_Tablespace_Aux_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-index}',
CONST_Tablespace_Aux_Index,
$sTemplate
);
$this->pgsqlRunScript($sTemplate, false);
}
public function recreateFunction()
{
// re-run the functions
info('Recreate Functions');
$this->createSqlFunctions();
}
public function createPartitionTables()
{
info('Create Partition Tables');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
$sTemplate = $this->replaceTablespace(
'{ts:address-data}',
CONST_Tablespace_Address_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:address-index}',
CONST_Tablespace_Address_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:search-data}',
CONST_Tablespace_Search_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:search-index}',
CONST_Tablespace_Search_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-data}',
CONST_Tablespace_Aux_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-index}',
CONST_Tablespace_Aux_Index,
$sTemplate
);
$this->pgsqlRunPartitionScript($sTemplate);
}
public function createPartitionFunctions()
{
info('Create Partition Functions');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
$this->pgsqlRunPartitionScript($sTemplate);
}
public function importWikipediaArticles()
{
$sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
$sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
if (file_exists($sWikiArticlesFile)) {
info('Importing wikipedia articles');
$this->pgsqlRunDropAndRestore($sWikiArticlesFile);
} else {
warn('wikipedia article dump file not found - places will have default importance');
}
if (file_exists($sWikiRedirectsFile)) {
info('Importing wikipedia redirects');
$this->pgsqlRunDropAndRestore($sWikiRedirectsFile);
} else {
warn('wikipedia redirect dump file not found - some place importance values may be missing');
}
echo ' finish wikipedia';
}
public function loadData($bDisableTokenPrecalc)
{
info('Drop old Data');
if ($this->oDB == null) $this->oDB =& getDB();
if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection));
echo '.';
$sSQL = 'select distinct partition from country_name';
$aPartitions = chksql($this->oDB->getCol($sSQL));
if (!$this->bNoPartitions) $aPartitions[] = 0;
foreach ($aPartitions as $sPartition) {
if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection));
echo '.';
}
// used by getorcreate_word_id to ignore frequent partial words
$sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
$sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
echo ".\n";
// pre-create the word list
if (!$bDisableTokenPrecalc) {
info('Loading word list');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
}
info('Load Data');
$sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry';
$aDBInstances = array();
$iLoadThreads = max(1, $this->iInstances - 1);
for ($i = 0; $i < $iLoadThreads; $i++) {
$aDBInstances[$i] =& getDB(true);
$sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
$sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
$sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')";
$sSQL .= ' and ST_IsValid(geometry)';
if ($this->sVerbose) echo "$sSQL\n";
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
fail(pg_last_error($aDBInstances[$i]->connection));
}
}
// last thread for interpolation lines
$aDBInstances[$iLoadThreads] =& getDB(true);
$sSQL = 'insert into location_property_osmline';
$sSQL .= ' (osm_id, address, linegeo)';
$sSQL .= ' SELECT osm_id, address, geometry from place where ';
$sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";
if ($this->sVerbose) echo "$sSQL\n";
if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) {
fail(pg_last_error($aDBInstances[$iLoadThreads]->connection));
}
$bFailed = false;
for ($i = 0; $i <= $iLoadThreads; $i++) {
while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) {
$resultStatus = pg_result_status($hPGresult);
// PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK,
// PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE,
// PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR
echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n";
if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) {
$resultError = pg_result_error($hPGresult);
echo '-- error text ' . $i . ': ' . $resultError . "\n";
$bFailed = true;
}
}
}
if ($bFailed) {
fail('SQL errors loading placex and/or location_property_osmline tables');
}
echo "\n";
info('Reanalysing database');
$this->pgsqlRunScript('ANALYSE');
$sDatabaseDate = getDatabaseDate($this->oDB);
pg_query($this->oDB->connection, 'TRUNCATE import_status');
if ($sDatabaseDate === false) {
warn('could not determine database date.');
} else {
$sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
pg_query($this->oDB->connection, $sSQL);
echo "Latest data imported from $sDatabaseDate.\n";
}
}
public function importTigerData()
{
info('Import Tiger data');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:aux-data}',
CONST_Tablespace_Aux_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-index}',
CONST_Tablespace_Aux_Index,
$sTemplate
);
$this->pgsqlRunScript($sTemplate, false);
$aDBInstances = array();
for ($i = 0; $i < $this->iInstances; $i++) {
$aDBInstances[$i] =& getDB(true);
}
foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) {
echo $sFile.': ';
$hFile = fopen($sFile, 'r');
$sSQL = fgets($hFile, 100000);
$iLines = 0;
while (true) {
for ($i = 0; $i < $this->iInstances; $i++) {
if (!pg_connection_busy($aDBInstances[$i]->connection)) {
while (pg_get_result($aDBInstances[$i]->connection));
$sSQL = fgets($hFile, 100000);
if (!$sSQL) break 2;
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
$iLines++;
if ($iLines == 1000) {
echo '.';
$iLines = 0;
}
}
}
usleep(10);
}
fclose($hFile);
$bAnyBusy = true;
while ($bAnyBusy) {
$bAnyBusy = false;
for ($i = 0; $i < $this->iInstances; $i++) {
if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
}
usleep(10);
}
echo "\n";
}
info('Creating indexes on Tiger data');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:aux-data}',
CONST_Tablespace_Aux_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-index}',
CONST_Tablespace_Aux_Index,
$sTemplate
);
$this->pgsqlRunScript($sTemplate, false);
}
public function calculatePostcodes($bCMDResultAll)
{
info('Calculate Postcodes');
if ($this->oDB == null) $this->oDB =& getDB();
if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) {
fail(pg_last_error($this->oDB->connection));
}
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, country_code,";
$sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,";
$sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))';
$sSQL .= ' FROM placex';
$sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
$sSQL .= ' AND geometry IS NOT null';
$sSQL .= ' GROUP BY country_code, pc';
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
if (CONST_Use_Extra_US_Postcodes) {
// only add postcodes that are not yet available in OSM
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
$sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)';
$sSQL .= ' FROM us_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'us')";
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
}
// add missing postcodes for GB (if available)
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry";
$sSQL .= ' FROM gb_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'gb')";
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
if (!$bCMDResultAll) {
$sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
$sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
}
$sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
$sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
}
public function index($bIndexNoanalyse)
{
$sOutputFile = '';
$sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P '
.$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec'];
}
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$sBaseCmd .= ' -U ' . $this->aDSNInfo['username'];
}
$aProcEnv = null;
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
}
info('Index ranks 0 - 4');
$iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv);
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
info('Index ranks 5 - 25');
$iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv);
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
info('Index ranks 26 - 30');
$iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv);
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
info('Index postcodes');
if ($this->oDB == null) $this->oDB =& getDB();
$sSQL = 'UPDATE location_postcode SET indexed_status = 0';
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
}
public function createSearchIndices()
{
info('Create Search indices');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:address-index}',
CONST_Tablespace_Address_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:search-index}',
CONST_Tablespace_Search_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-index}',
CONST_Tablespace_Aux_Index,
$sTemplate
);
$this->pgsqlRunScript($sTemplate);
}
public function createCountryNames()
{
info('Create search index for default country names');
$this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')");
$this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')");
$this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x');
$this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x");
$sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),'
.'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k ';
if (CONST_Languages) {
$sSQL .= 'in ';
$sDelim = '(';
foreach (explode(',', CONST_Languages) as $sLang) {
$sSQL .= $sDelim."'name:$sLang'";
$sDelim = ',';
}
$sSQL .= ')';
} else {
// all include all simple name tags
$sSQL .= "like 'name:%'";
}
$sSQL .= ') v';
$this->pgsqlRunScript($sSQL);
}
public function drop()
{
info('Drop tables only required for updates');
// The implementation is potentially a bit dangerous because it uses
// a positive selection of tables to keep, and deletes everything else.
// Including any tables that the unsuspecting user might have manually
// created. USE AT YOUR OWN PERIL.
// tables we want to keep. everything else goes.
$aKeepTables = array(
'*columns',
'import_polygon_*',
'import_status',
'place_addressline',
'location_postcode',
'location_property*',
'placex',
'search_name',
'seq_*',
'word',
'query_log',
'new_query_log',
'spatial_ref_sys',
'country_name',
'place_classtype_*'
);
if ($this->oDB = null) $this->oDB =& getDB();
$aDropTables = array();
$aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"));
foreach ($aHaveTables as $sTable) {
$bFound = false;
foreach ($aKeepTables as $sKeep) {
if (fnmatch($sKeep, $sTable)) {
$bFound = true;
break;
}
}
if (!$bFound) array_push($aDropTables, $sTable);
}
foreach ($aDropTables as $sDrop) {
if ($this->sVerbose) echo "dropping table $sDrop\n";
@pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE");
// ignore warnings/errors as they might be caused by a table having
// been deleted already by CASCADE
}
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n";
unlink(CONST_Osm2pgsql_Flatnode_File);
}
}
private function pgsqlRunDropAndRestore($sDumpFile)
{
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
$sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sCMD .= ' -h ' . $this->aDSNInfo['hostspec'];
}
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$sCMD .= ' -U ' . $this->aDSNInfo['username'];
}
$aProcEnv = null;
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
}
$iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)"
}
private function pgsqlRunScript($sScript, $bfatal = true)
{
runSQLScript(
$sScript,
$bfatal,
$this->sVerbose,
$this->sIgnoreErrors
);
}
private function createSqlFunctions()
{
$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
$sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
if ($this->bEnableDiffUpdates) {
$sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
}
if ($this->bEnableDebugStatements) {
$sTemplate = str_replace('--DEBUG:', '', $sTemplate);
}
if (CONST_Limit_Reindexing) {
$sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
}
if (!CONST_Use_US_Tiger_Data) {
$sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate);
}
if (!CONST_Use_Aux_Location_data) {
$sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate);
}
$this->pgsqlRunScript($sTemplate);
}
private function pgsqlRunPartitionScript($sTemplate)
{
if ($this->oDB == null) $this->oDB =& getDB();
$sSQL = 'select distinct partition from country_name';
$aPartitions = chksql($this->oDB->getCol($sSQL));
if (!$this->bNoPartitions) $aPartitions[] = 0;
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
foreach ($aMatches as $aMatch) {
$sResult = '';
foreach ($aPartitions as $sPartitionName) {
$sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
}
$sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
}
$this->pgsqlRunScript($sTemplate);
}
private function pgsqlRunScriptFile($sFilename)
{
if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
$sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'];
if (!$this->sVerbose) {
$sCMD .= ' -q';
}
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sCMD .= ' -h ' . $this->aDSNInfo['hostspec'];
}
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$sCMD .= ' -U ' . $this->aDSNInfo['username'];
}
$aProcEnv = null;
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
}
$ahGzipPipes = null;
if (preg_match('/\\.gz$/', $sFilename)) {
$aDescriptors = array(
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a')
);
$hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes);
if (!is_resource($hGzipProcess)) fail('unable to start zcat');
$aReadPipe = $ahGzipPipes[1];
fclose($ahGzipPipes[0]);
} else {
$sCMD .= ' -f '.$sFilename;
$aReadPipe = array('pipe', 'r');
}
$aDescriptors = array(
0 => $aReadPipe,
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a')
);
$ahPipes = null;
$hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
if (!is_resource($hProcess)) fail('unable to start pgsql');
// TODO: error checking
while (!feof($ahPipes[1])) {
echo fread($ahPipes[1], 4096);
}
fclose($ahPipes[1]);
$iReturn = proc_close($hProcess);
if ($iReturn > 0) {
fail("pgsql returned with error code ($iReturn)");
}
if ($ahGzipPipes) {
fclose($ahGzipPipes[1]);
proc_close($hGzipProcess);
}
}
private function replaceTablespace($sTemplate, $sTablespace, $sSql)
{
if ($sTablespace) {
$sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql);
} else {
$sSql = str_replace($sTemplate, '', $sSql);
}
return $sSql;
}
}

190
utils/update.php
View File

@@ -1,66 +1,42 @@
#!@PHP_BIN@ -Cq
<?php
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
require_once(CONST_BasePath.'/lib/init-cmd.php');
include_once(CONST_BasePath.'/lib/setup_functions.php');
require_once dirname(dirname(__FILE__)) . '/settings/settings.php';
require_once CONST_BasePath . '/lib/init-cmd.php';
require_once CONST_BasePath . '/lib/setup_functions.php';
require_once CONST_BasePath . '/lib/SetupClass.php';
ini_set('memory_limit', '800M');
use Nominatim\Setup\SetupFunctions as SetupFunctions;
# (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
$aCMDOptions = createUpdateArgvArray();
$aCMDOptions
= array(
'Import / update / index osm data',
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
);
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
if (!isset($aResult['index-instances'])) {
$aResult['index-instances'] = 1;
}
if (!isset($aResult['index-rank'])) {
$aResult['index-rank'] = 0;
}
date_default_timezone_set('Etc/UTC');
$oDB =& getDB();
$oDB = &getDB();
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) {
$aDSNInfo['port'] = 5432;
}
// cache memory to be used by osm2pgsql, should not be more than the available memory
$iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
$iCacheMemory = (isset($aResult['osm2pgsql-cache']) ? $aResult['osm2pgsql-cache'] : 2000);
if ($iCacheMemory + 500 > getTotalMemoryMB()) {
$iCacheMemory = getCacheMemoryMB();
echo "WARNING: resetting cache memory to $iCacheMemory\n";
}
$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary . ' -klas --number-processes 1 -C ' . $iCacheMemory . ' -O gazetteer -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'];
if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
$sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username'];
}
@@ -73,12 +49,12 @@ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
}
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
$sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
$sOsm2pgsqlCmd .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File;
}
if ($aResult['init-updates']) {
// sanity check that the replication URL is correct
$sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt');
$sBaseState = file_get_contents(CONST_Replication_Url . '/state.txt');
if ($sBaseState === false) {
echo "\nCannot find state.txt file at the configured replication URL.\n";
echo "Does the URL point to a directory containing OSM update data?\n\n";
@@ -92,7 +68,7 @@ if ($aResult['init-updates']) {
fail('CONST_Pyosmium_Binary not configured');
}
$aOutput = 0;
$sCmd = CONST_Pyosmium_Binary.' --help';
$sCmd = CONST_Pyosmium_Binary . ' --help';
exec($sCmd, $aOutput, $iRet);
if ($iRet != 0) {
echo "Cannot execute pyosmium-get-changes.\n";
@@ -100,19 +76,23 @@ if ($aResult['init-updates']) {
echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n";
fail('pyosmium-get-changes not found or not usable');
}
if (!$aResult['no-update-functions']) {
createFunctions($aCMDResult);
// instatiate setupClass to use the function therein
// instantiate Setup class
$cSetup = new SetupFunctions($aResult);
$cSetup->createFunctions();
}
$sDatabaseDate = getDatabaseDate($oDB);
if ($sDatabaseDate === false) {
fail('Cannot determine date of database.');
}
$sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60));
$sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3 * 60 * 60));
// get the appropriate state id
$aOutput = 0;
$sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url;
$sCmd = CONST_Pyosmium_Binary . ' -D ' . $sWindBack . ' --server ' . CONST_Replication_Url;
exec($sCmd, $aOutput, $iRet);
if ($iRet != 0 || $aOutput[0] == 'None') {
fail('Error running pyosmium tools');
@@ -120,7 +100,7 @@ if ($aResult['init-updates']) {
pg_query($oDB->connection, 'TRUNCATE import_status');
$sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('";
$sSQL .= $sDatabaseDate."',".$aOutput[0].', true)';
$sSQL .= $sDatabaseDate . "'," . $aOutput[0] . ', true)';
if (!pg_query($oDB->connection, $sSQL)) {
fail('Could not enter sequence into database.');
}
@@ -135,7 +115,7 @@ if ($aResult['check-for-updates']) {
fail('Updates not set up. Please run ./utils/update.php --init-updates.');
}
system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet);
system(CONST_BasePath . '/utils/check_server_for_updates.py ' . CONST_Replication_Url . ' ' . $aLastState['sequence_id'], $iRet);
exit($iRet);
}
@@ -148,8 +128,8 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
}
// Import the file
$sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
echo $sCMD."\n";
$sCMD = $sOsm2pgsqlCmd . ' ' . $sNextFile;
echo $sCMD . "\n";
$iErrorLevel = runWithEnv($sCMD, $aProcEnv);
if ($iErrorLevel) {
@@ -161,35 +141,35 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
if ($aResult['calculate-postcodes']) {
info('Update postcodes centroids');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/update-postcodes.sql');
runSQLScript($sTemplate, true, true);
}
$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
$sTemporaryFile = CONST_BasePath . '/data/osmosischange.osc';
$bHaveDiff = false;
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
$sContentURL = '';
if (isset($aResult['import-node']) && $aResult['import-node']) {
if ($bUseOSMApi) {
$sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
$sContentURL = 'https://www.openstreetmap.org/api/0.6/node/' . $aResult['import-node'];
} else {
$sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
$sContentURL = 'https://overpass-api.de/api/interpreter?data=node(' . $aResult['import-node'] . ');out%20meta;';
}
}
if (isset($aResult['import-way']) && $aResult['import-way']) {
if ($bUseOSMApi) {
$sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
$sContentURL = 'https://www.openstreetmap.org/api/0.6/way/' . $aResult['import-way'] . '/full';
} else {
$sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
$sContentURL = 'https://overpass-api.de/api/interpreter?data=(way(' . $aResult['import-way'] . ');node(w););out%20meta;';
}
}
if (isset($aResult['import-relation']) && $aResult['import-relation']) {
if ($bUseOSMApi) {
$sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
$sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/' . $aResult['import-relation'] . '/full';
} else {
$sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
$sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel(' . $aResult['import-relation'] . ');way(r);node(w));node(r));out%20meta;';
}
}
@@ -200,8 +180,8 @@ if ($sContentURL) {
if ($bHaveDiff) {
// import generated change file
$sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
echo $sCMD."\n";
$sCMD = $sOsm2pgsqlCmd . ' ' . $sTemporaryFile;
echo $sCMD . "\n";
$iErrorLevel = runWithEnv($sCMD, $aProcEnv);
if ($iErrorLevel) {
fail("osm2pgsql exited with error level $iErrorLevel\n");
@@ -209,7 +189,7 @@ if ($bHaveDiff) {
}
if ($aResult['deduplicate']) {
$oDB =& getDB();
$oDB = &getDB();
if (getPostgresVersion($oDB) < 9.3) {
fail('ERROR: deduplicate is only currently supported in postgresql 9.3');
@@ -221,7 +201,7 @@ if ($aResult['deduplicate']) {
// we don't care about empty search_name_* partitions, they can't contain mentions of duplicates
foreach ($aPartitions as $i => $sPartition) {
$sSQL = 'select count(*) from search_name_'.$sPartition;
$sSQL = 'select count(*) from search_name_' . $sPartition;
$nEntries = chksql($oDB->getOne($sSQL));
if ($nEntries == 0) {
unset($aPartitions[$i]);
@@ -233,11 +213,14 @@ if ($aResult['deduplicate']) {
$sSQL .= ' group by word_token having count(*) > 1 order by word_token';
$aDuplicateTokens = chksql($oDB->getAll($sSQL));
foreach ($aDuplicateTokens as $aToken) {
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
echo 'Deduping '.$aToken['word_token']."\n";
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') {
continue;
}
echo 'Deduping ' . $aToken['word_token'] . "\n";
$sSQL = 'select word_id,';
$sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num';
$sSQL .= " from word where word_token = '".$aToken['word_token'];
$sSQL .= " from word where word_token = '" . $aToken['word_token'];
$sSQL .= "' and class is null and type is null and country_code is null order by num desc";
$aTokenSet = chksql($oDB->getAll($sSQL));
@@ -246,34 +229,34 @@ if ($aResult['deduplicate']) {
foreach ($aTokenSet as $aRemove) {
$sSQL = 'update search_name set';
$sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),';
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
$sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
$sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . '),';
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
$sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']';
chksql($oDB->query($sSQL));
$sSQL = 'update search_name set';
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
$sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']';
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
$sSQL .= ' where nameaddress_vector @> ARRAY[' . $aRemove['word_id'] . ']';
chksql($oDB->query($sSQL));
$sSQL = 'update location_area_country set';
$sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
$sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
$sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
$sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']';
chksql($oDB->query($sSQL));
foreach ($aPartitions as $sPartition) {
$sSQL = 'update search_name_'.$sPartition.' set';
$sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')';
$sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
$sSQL = 'update search_name_' . $sPartition . ' set';
$sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
$sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']';
chksql($oDB->query($sSQL));
$sSQL = 'update location_area_country set';
$sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
$sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
$sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
$sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']';
chksql($oDB->query($sSQL));
}
$sSQL = 'delete from word where word_id = '.$aRemove['word_id'];
$sSQL = 'delete from word where word_id = ' . $aRemove['word_id'];
chksql($oDB->query($sSQL));
}
}
@@ -281,12 +264,12 @@ if ($aResult['deduplicate']) {
if ($aResult['recompute-word-counts']) {
info('Recompute frequency of full-word search terms');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/words_from_search_name.sql');
runSQLScript($sTemplate, true, true);
}
if ($aResult['index']) {
$sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank'];
$sCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'] . ' -r ' . $aResult['index-rank'];
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
$sCmd .= ' -H ' . $aDSNInfo['hostspec'];
}
@@ -301,13 +284,13 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
//
if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
fail('Error: Update interval too low for download.geofabrik.de. ' .
"Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");
"Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");
}
$sImportFile = CONST_InstallPath.'/osmosischange.osc';
$sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size;
$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
$sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
$sImportFile = CONST_InstallPath . '/osmosischange.osc';
$sCMDDownload = CONST_Pyosmium_Binary . ' --server ' . CONST_Replication_Url . ' -o ' . $sImportFile . ' -s ' . CONST_Replication_Max_Diff_size;
$sCMDImport = $sOsm2pgsqlCmd . ' ' . $sImportFile;
$sCMDIndex = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'];
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
$sCMDIndex .= ' -H ' . $aDSNInfo['hostspec'];
}
@@ -324,7 +307,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
exit(1);
}
echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n";
echo 'Currently at sequence ' . $aLastState['sequence_id'] . ' (' . $aLastState['lastimportdate'] . ') - ' . $aLastState['indexed'] . " indexed\n";
$sBatchEnd = $aLastState['lastimportdate'];
$iEndSequence = $aLastState['sequence_id'];
@@ -347,23 +330,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
if (file_exists($sImportFile)) {
unlink($sImportFile);
}
exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult);
exec($sCMDDownload . ' -I ' . $iNextSeq, $aOutput, $iResult);
if ($iResult == 3) {
echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
echo 'No new updates. Sleeping for ' . CONST_Replication_Recheck_Interval . " sec.\n";
sleep(CONST_Replication_Recheck_Interval);
} elseif ($iResult != 0) {
echo 'ERROR: updates failed.';
exit($iResult);
} else {
$iEndSequence = (int)$aOutput[0];
$iEndSequence = (int) $aOutput[0];
}
} while ($iResult);
// get the newest object from the diff file
$sBatchEnd = 0;
$iRet = 0;
exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
exec(CONST_BasePath . '/utils/osm_file_date.py ' . $sImportFile, $sBatchEnd, $iRet);
if ($iRet == 5) {
echo "Diff file is empty. skipping import.\n";
if (!$aResult['import-osmosis-all']) {
@@ -379,7 +362,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
// Import the file
$fCMDStartTime = time();
echo $sCMDImport."\n";
echo $sCMDImport . "\n";
unset($sJunk);
$iErrorLevel = runWithEnv($sCMDImport, $aProcEnv);
if ($iErrorLevel) {
@@ -392,8 +375,8 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
$sSQL = 'INSERT INTO import_osmosis_log';
$sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
$sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
$sSQL .= date('Y-m-d H:i:s')."','import')";
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','";
$sSQL .= date('Y-m-d H:i:s') . "','import')";
var_Dump($sSQL);
chksql($oDB->query($sSQL));
@@ -401,7 +384,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
$sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence";
var_Dump($sSQL);
chksql($oDB->query($sSQL));
echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
echo date('Y-m-d H:i:s') . " Completed download step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n";
}
// Index file
@@ -419,18 +402,21 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
$sSQL = 'INSERT INTO import_osmosis_log';
$sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
$sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
$sSQL .= date('Y-m-d H:i:s')."','index')";
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','";
$sSQL .= date('Y-m-d H:i:s') . "','index')";
var_Dump($sSQL);
$oDB->query($sSQL);
echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
echo date('Y-m-d H:i:s') . " Completed index step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n";
$sSQL = 'update import_status set indexed = true';
$oDB->query($sSQL);
}
$fDuration = time() - $fStartTime;
echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n";
if (!$aResult['import-osmosis-all']) exit(0);
echo date('Y-m-d H:i:s') . " Completed all for $sBatchEnd in " . round($fDuration / 60, 2) . " minutes\n";
if (!$aResult['import-osmosis-all']) {
exit(0);
}
}
}