hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-15 15:34:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

small fixes on setup.php and a bring update.php to work

Browse Source
This commit is contained in:
ThomasBarris
2018-08-31 21:31:38 +02:00
parent b2f3cfde0b
commit a3b4f80c99
5 changed files with 1297 additions and 1160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@@ -111,7 +111,6 @@ set(CUSTOMFILES
utils/query.php
utils/server_compare.php
utils/setup.php
utils/setupClass.php
utils/specialphrases.php
utils/update.php
utils/warm.php

230
utils/setupClass.php → lib/SetupClass.php Executable file → Normal file
View File

@@ -1,5 +1,7 @@
<?php
namespace Nominatim\Setup;
class SetupFunctions
{
protected $iCacheMemory; // set in constructor
@@ -14,13 +16,13 @@ class SetupFunctions
protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData
// pgsqlRunPartitionScript, calculatePostcodes, ..if no already set
public function __construct($aCMDResult) {
public function __construct($aCMDResult)
{
// by default, use all but one processor, but never more than 15.
$this->iInstances = isset($aCMDResult['threads'])
? $aCMDResult['threads']
: (min(16, getProcessorCount()) - 1);
if ($this->iInstances < 1) {
$this->iInstances = 1;
warn('resetting threads to ' . $this->iInstances);
@@ -37,8 +39,10 @@ class SetupFunctions
info('module path: ' . $this->sModulePath);
// prepares DB for import or update, sets the Data Source Name
$this->aDSNInfo = DB::parseDSN(CONST_Database_DSN);
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
$this->aDSNInfo = \DB::parseDSN(CONST_Database_DSN);
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) {
$this->aDSNInfo['port'] = 5432;
}
// setting member variables based on command line options stored in $aCMDResult
$this->sVerbose = $aCMDResult['verbose'];
@@ -51,8 +55,8 @@ class SetupFunctions
public function createDB()
{
info('Create DB');
$sDB = DB::connect(CONST_Database_DSN, false);
if (!PEAR::isError($sDB)) {
$sDB = \DB::connect(CONST_Database_DSN, false);
if (!\PEAR::isError($sDB)) {
fail('database already exists (' . CONST_Database_DSN . ')');
}
@@ -71,7 +75,10 @@ class SetupFunctions
}
$result = runWithEnv($sCreateDBCmd, $aProcEnv);
if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd);
if ($result != 0) {
fail('Error executing external command: ' . $sCreateDBCmd);
}
}
public function setupDB()
@@ -99,7 +106,6 @@ class SetupFunctions
warn('Postgresql is too old. extratags and namedetails API not available.');
}
$fPostgisVersion = getPostgisVersion($this->oDB);
echo 'Postgis version found: ' . $fPostgisVersion . "\n";
@@ -129,7 +135,6 @@ class SetupFunctions
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_osm_grid.sql.gz');
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_table.sql');
if (file_exists(CONST_BasePath . '/data/gb_postcode_data.sql.gz')) {
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_data.sql.gz');
} else {
@@ -163,20 +168,26 @@ class SetupFunctions
fail("osm2pgsql not found in '$osm2pgsql'");
}
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
$osm2pgsql .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File;
}
if (CONST_Tablespace_Osm2pgsql_Data)
if (CONST_Tablespace_Osm2pgsql_Data) {
$osm2pgsql .= ' --tablespace-slim-data ' . CONST_Tablespace_Osm2pgsql_Data;
if (CONST_Tablespace_Osm2pgsql_Index)
}
if (CONST_Tablespace_Osm2pgsql_Index) {
$osm2pgsql .= ' --tablespace-slim-index ' . CONST_Tablespace_Osm2pgsql_Index;
if (CONST_Tablespace_Place_Data)
}
if (CONST_Tablespace_Place_Data) {
$osm2pgsql .= ' --tablespace-main-data ' . CONST_Tablespace_Place_Data;
if (CONST_Tablespace_Place_Index)
}
if (CONST_Tablespace_Place_Index) {
$osm2pgsql .= ' --tablespace-main-index ' . CONST_Tablespace_Place_Index;
}
$osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
$osm2pgsql .= ' -C ' . $this->iCacheMemory;
$osm2pgsql .= ' -P ' . $this->aDSNInfo['port'];
@@ -192,7 +203,10 @@ class SetupFunctions
}
$osm2pgsql .= ' -d ' . $this->aDSNInfo['database'] . ' ' . $sOSMFile;
runWithEnv($osm2pgsql, $aProcEnv);
if ($this->oDB == null) $this->oDB =& getDB();
if ($this->oDB == null) {
$this->oDB = &getDB();
}
if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) {
fail('No Data');
}
@@ -245,13 +259,6 @@ class SetupFunctions
$this->pgsqlRunScript($sTemplate, false);
}
public function recreateFunction()
{
// re-run the functions
info('Recreate Functions');
$this->createSqlFunctions();
}
public function createPartitionTables()
{
info('Create Partition Tables');
@@ -327,34 +334,72 @@ class SetupFunctions
{
info('Drop old Data');
if ($this->oDB == null) $this->oDB =& getDB();
if ($this->oDB == null) {
$this->oDB = &getDB();
}
if (!pg_query($this->oDB->connection, 'TRUNCATE word')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection));
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.';
if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.';
if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.';
$sSQL = 'select distinct partition from country_name';
$aPartitions = chksql($this->oDB->getCol($sSQL));
if (!$this->bNoPartitions) $aPartitions[] = 0;
if (!$this->bNoPartitions) {
$aPartitions[] = 0;
}
foreach ($aPartitions as $sPartition) {
if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_' . $sPartition)) {
fail(pg_last_error($this->oDB->connection));
}
echo '.';
}
@@ -382,7 +427,10 @@ class SetupFunctions
$sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
$sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')";
$sSQL .= ' and ST_IsValid(geometry)';
if ($this->sVerbose) echo "$sSQL\n";
if ($this->sVerbose) {
echo "$sSQL\n";
}
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
fail(pg_last_error($aDBInstances[$i]->connection));
}
@@ -394,7 +442,10 @@ class SetupFunctions
$sSQL .= ' (osm_id, address, linegeo)';
$sSQL .= ' SELECT osm_id, address, geometry from place where ';
$sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";
if ($this->sVerbose) echo "$sSQL\n";
if ($this->sVerbose) {
echo "$sSQL\n";
}
if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) {
fail(pg_last_error($aDBInstances[$iLoadThreads]->connection));
}
@@ -465,8 +516,14 @@ class SetupFunctions
if (!pg_connection_busy($aDBInstances[$i]->connection)) {
while (pg_get_result($aDBInstances[$i]->connection));
$sSQL = fgets($hFile, 100000);
if (!$sSQL) break 2;
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
if (!$sSQL) {
break 2;
}
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
$iLines++;
if ($iLines == 1000) {
echo '.';
@@ -482,7 +539,10 @@ class SetupFunctions
while ($bAnyBusy) {
$bAnyBusy = false;
for ($i = 0; $i < $this->iInstances; $i++) {
if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
if (pg_connection_busy($aDBInstances[$i]->connection)) {
$bAnyBusy = true;
}
}
usleep(10);
}
@@ -508,12 +568,14 @@ class SetupFunctions
public function calculatePostcodes($bCMDResultAll)
{
info('Calculate Postcodes');
if ($this->oDB == null) $this->oDB =& getDB();
if ($this->oDB == null) {
$this->oDB = &getDB();
}
if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) {
fail(pg_last_error($this->oDB->connection));
}
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, country_code,";
@@ -537,7 +599,10 @@ class SetupFunctions
$sSQL .= ' FROM us_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'us')";
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
}
// add missing postcodes for GB (if available)
@@ -547,7 +612,9 @@ class SetupFunctions
$sSQL .= ' FROM gb_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'gb')";
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
if (!$bCMDResultAll) {
$sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
@@ -585,22 +652,34 @@ class SetupFunctions
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
if (!$bIndexNoanalyse) {
$this->pgsqlRunScript('ANALYSE');
}
info('Index ranks 5 - 25');
$iStatus = runWithEnv($sBaseCmd . ' -r 5 -R 25', $aProcEnv);
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
if (!$bIndexNoanalyse) {
$this->pgsqlRunScript('ANALYSE');
}
info('Index ranks 26 - 30');
$iStatus = runWithEnv($sBaseCmd . ' -r 26', $aProcEnv);
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
info('Index postcodes');
if ($this->oDB == null) $this->oDB =& getDB();
if ($this->oDB == null) {
$this->oDB = &getDB();
}
$sSQL = 'UPDATE location_postcode SET indexed_status = 0';
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
}
public function createSearchIndices()
@@ -677,10 +756,13 @@ class SetupFunctions
'new_query_log',
'spatial_ref_sys',
'country_name',
'place_classtype_*'
'place_classtype_*',
);
if ($this->oDB = null) $this->oDB =& getDB();
if ($this->oDB = null) {
$this->oDB = &getDB();
}
$aDropTables = array();
$aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"));
@@ -692,24 +774,36 @@ class SetupFunctions
break;
}
}
if (!$bFound) array_push($aDropTables, $sTable);
if (!$bFound) {
array_push($aDropTables, $sTable);
}
}
foreach ($aDropTables as $sDrop) {
if ($this->sVerbose) echo "dropping table $sDrop\n";
if ($this->sVerbose) {
echo "dropping table $sDrop\n";
}
@pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE");
// ignore warnings/errors as they might be caused by a table having
// been deleted already by CASCADE
}
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n";
if ($sVerbose) {
echo 'deleting ' . CONST_Osm2pgsql_Flatnode_File . "\n";
}
unlink(CONST_Osm2pgsql_Flatnode_File);
}
}
private function pgsqlRunDropAndRestore($sDumpFile)
{
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) {
$this->aDSNInfo['port'] = 5432;
}
$sCMD = 'pg_restore -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database'] . ' -Fc --clean ' . $sDumpFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sCMD .= ' -h ' . $this->aDSNInfo['hostspec'];
@@ -758,11 +852,15 @@ class SetupFunctions
private function pgsqlRunPartitionScript($sTemplate)
{
if ($this->oDB == null) $this->oDB =& getDB();
if ($this->oDB == null) {
$this->oDB = &getDB();
}
$sSQL = 'select distinct partition from country_name';
$aPartitions = chksql($this->oDB->getCol($sSQL));
if (!$this->bNoPartitions) $aPartitions[] = 0;
if (!$this->bNoPartitions) {
$aPartitions[] = 0;
}
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
foreach ($aMatches as $aMatch) {
@@ -778,7 +876,9 @@ class SetupFunctions
private function pgsqlRunScriptFile($sFilename)
{
if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
if (!file_exists($sFilename)) {
fail('unable to find ' . $sFilename);
}
$sCMD = 'psql -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database'];
if (!$this->sVerbose) {
@@ -799,10 +899,13 @@ class SetupFunctions
$aDescriptors = array(
0 => array('pipe', 'r'),
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a')
2 => array('file', '/dev/null', 'a'),
);
$hGzipProcess = proc_open('zcat ' . $sFilename, $aDescriptors, $ahGzipPipes);
if (!is_resource($hGzipProcess)) fail('unable to start zcat');
if (!is_resource($hGzipProcess)) {
fail('unable to start zcat');
}
$aReadPipe = $ahGzipPipes[1];
fclose($ahGzipPipes[0]);
} else {
@@ -812,11 +915,14 @@ class SetupFunctions
$aDescriptors = array(
0 => $aReadPipe,
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a')
2 => array('file', '/dev/null', 'a'),
);
$ahPipes = null;
$hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
if (!is_resource($hProcess)) fail('unable to start pgsql');
if (!is_resource($hProcess)) {
fail('unable to start pgsql');
}
// TODO: error checking
while (!feof($ahPipes[1])) {
echo fread($ahPipes[1], 4096);

46
lib/setup_functions.php
View File

@@ -38,6 +38,8 @@ function checkModulePresence()
return $bResult;
}
// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
// create and array
function createSetupArgvArray()
{
$aCMDOptions
@@ -78,3 +80,47 @@ function createSetupArgvArray()
);
return $aCMDOptions;
}
function createUpdateArgvArray()
{
$aCMDOptions
= array(
'Import / update / index osm data',
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
array('create-functions', '', 0, 1, 1, 1, 'bool', 'Create functions'),
array('enable-diff-updates', '', 0, 1, 1, 1, 'bool', 'Turn on the code required to make diff updates work'),
array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
);
return $aCMDOptions;
}

18
utils/setup.php
View File

@@ -1,23 +1,22 @@
#!@PHP_BIN@ -Cq
<?php
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
require_once(CONST_BasePath.'/lib/init-cmd.php');
include_once(CONST_InstallPath.'/utils/setupClass.php');
require_once dirname(dirname(__FILE__)) . '/settings/settings.php';
require_once CONST_BasePath . '/lib/init-cmd.php';
require_once CONST_BasePath . '/lib/SetupClass.php';
// ->indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt
// ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB()
include_once(CONST_BasePath.'/lib/setup_functions.php');
include_once(CONST_BasePath.'/lib/setup_functions.php');
require_once CONST_BasePath . '/lib/setup_functions.php';
ini_set('memory_limit', '800M');
use Nominatim\Setup\SetupFunctions as SetupFunctions;
$aCMDOptions = createSetupArgvArray();
// $aCMDOptions passed to getCmdOpt by reference
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
$bDidSomething = false;
//*******************************************************
@@ -38,13 +37,14 @@ if ($aCMDResult['osmosis-init']) {
// ******************************************************
// instantiate Setup class
$cSetup = new SetupFunctions($aCMDResult);
// *******************************************************
// go through complete process if 'all' is selected or start selected functions
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->createDB();
}
// *******************************************************
// go through complete process if 'all' is selected or start selected functions
if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->setupDB();
@@ -68,7 +68,7 @@ if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->createTables();
$cSetup -> recreateFunction();
$cSetup->createFunctions();
}
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {

72
utils/update.php
View File

@@ -1,58 +1,34 @@
#!@PHP_BIN@ -Cq
<?php
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
require_once(CONST_BasePath.'/lib/init-cmd.php');
include_once(CONST_BasePath.'/lib/setup_functions.php');
require_once dirname(dirname(__FILE__)) . '/settings/settings.php';
require_once CONST_BasePath . '/lib/init-cmd.php';
require_once CONST_BasePath . '/lib/setup_functions.php';
require_once CONST_BasePath . '/lib/SetupClass.php';
ini_set('memory_limit', '800M');
use Nominatim\Setup\SetupFunctions as SetupFunctions;
# (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
$aCMDOptions = createUpdateArgvArray();
$aCMDOptions
= array(
'Import / update / index osm data',
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
);
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
if (!isset($aResult['index-instances'])) {
$aResult['index-instances'] = 1;
}
if (!isset($aResult['index-rank'])) {
$aResult['index-rank'] = 0;
}
date_default_timezone_set('Etc/UTC');
$oDB = &getDB();
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) {
$aDSNInfo['port'] = 5432;
}
// cache memory to be used by osm2pgsql, should not be more than the available memory
$iCacheMemory = (isset($aResult['osm2pgsql-cache']) ? $aResult['osm2pgsql-cache'] : 2000);
@@ -100,8 +76,12 @@ if ($aResult['init-updates']) {
echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n";
fail('pyosmium-get-changes not found or not usable');
}
if (!$aResult['no-update-functions']) {
createFunctions($aCMDResult);
// instatiate setupClass to use the function therein
// instantiate Setup class
$cSetup = new SetupFunctions($aResult);
$cSetup->createFunctions();
}
$sDatabaseDate = getDatabaseDate($oDB);
@@ -233,7 +213,10 @@ if ($aResult['deduplicate']) {
$sSQL .= ' group by word_token having count(*) > 1 order by word_token';
$aDuplicateTokens = chksql($oDB->getAll($sSQL));
foreach ($aDuplicateTokens as $aToken) {
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') {
continue;
}
echo 'Deduping ' . $aToken['word_token'] . "\n";
$sSQL = 'select word_id,';
$sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num';
@@ -431,6 +414,9 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
$fDuration = time() - $fStartTime;
echo date('Y-m-d H:i:s') . " Completed all for $sBatchEnd in " . round($fDuration / 60, 2) . " minutes\n";
if (!$aResult['import-osmosis-all']) exit(0);
if (!$aResult['import-osmosis-all']) {
exit(0);
}
}
}