mirror of
https://github.com/osm-search/Nominatim.git
synced 2026-02-16 15:47:58 +00:00
tabs-to-spaces
This commit is contained in:
Marc Tobias Metten
@@ -1,53 +1,53 @@
|
||||
#!/usr/bin/php -Cq
|
||||
<?php
|
||||
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
|
||||
$aCMDOptions = array(
|
||||
"Manage service blocks / restrictions",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
|
||||
array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),
|
||||
array('flush', '', 0, 1, 0, 0, 'bool', 'Flush all blocks / stats'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
|
||||
$aCMDOptions = array(
|
||||
"Manage service blocks / restrictions",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
|
||||
array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),
|
||||
array('flush', '', 0, 1, 0, 0, 'bool', 'Flush all blocks / stats'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
|
||||
|
||||
$m = getBucketMemcache();
|
||||
if (!$m)
|
||||
{
|
||||
echo "ERROR: Bucket memcache is not configured\n";
|
||||
exit;
|
||||
}
|
||||
$m = getBucketMemcache();
|
||||
if (!$m)
|
||||
{
|
||||
echo "ERROR: Bucket memcache is not configured\n";
|
||||
exit;
|
||||
}
|
||||
|
||||
if ($aResult['list'])
|
||||
{
|
||||
$iCurrentSleeping = $m->get('sleepCounter');
|
||||
echo "\n Sleeping blocks count: $iCurrentSleeping\n";
|
||||
if ($aResult['list'])
|
||||
{
|
||||
$iCurrentSleeping = $m->get('sleepCounter');
|
||||
echo "\n Sleeping blocks count: $iCurrentSleeping\n";
|
||||
|
||||
$aBlocks = getBucketBlocks();
|
||||
echo "\n";
|
||||
printf(" %-40s | %12s | %7s | %13s | %31s | %8s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Block Time", "Sleeping");
|
||||
printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s-|-%'-8s\n", "", "", "", "", "", "");
|
||||
foreach($aBlocks as $sKey => $aDetails)
|
||||
{
|
||||
printf(" %-40s | %12s | %7s | %13s | %31s | %8s\n", $sKey, $aDetails['totalBlocks'],
|
||||
(int)$aDetails['currentBucketSize'], $aDetails['currentlyBlocked']?'Y':'N',
|
||||
date("r", $aDetails['lastBlockTimestamp']), $aDetails['isSleeping']?'Y':'N');
|
||||
}
|
||||
echo "\n";
|
||||
}
|
||||
$aBlocks = getBucketBlocks();
|
||||
echo "\n";
|
||||
printf(" %-40s | %12s | %7s | %13s | %31s | %8s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Block Time", "Sleeping");
|
||||
printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s-|-%'-8s\n", "", "", "", "", "", "");
|
||||
foreach($aBlocks as $sKey => $aDetails)
|
||||
{
|
||||
printf(" %-40s | %12s | %7s | %13s | %31s | %8s\n", $sKey, $aDetails['totalBlocks'],
|
||||
(int)$aDetails['currentBucketSize'], $aDetails['currentlyBlocked']?'Y':'N',
|
||||
date("r", $aDetails['lastBlockTimestamp']), $aDetails['isSleeping']?'Y':'N');
|
||||
}
|
||||
echo "\n";
|
||||
}
|
||||
|
||||
if ($aResult['delete'])
|
||||
{
|
||||
$m->set('sleepCounter', 0);
|
||||
clearBucketBlocks();
|
||||
}
|
||||
if ($aResult['delete'])
|
||||
{
|
||||
$m->set('sleepCounter', 0);
|
||||
clearBucketBlocks();
|
||||
}
|
||||
|
||||
if ($aResult['flush'])
|
||||
{
|
||||
$m->flush();
|
||||
}
|
||||
if ($aResult['flush'])
|
||||
{
|
||||
$m->flush();
|
||||
}
|
||||
|
||||
@@ -1,36 +1,36 @@
|
||||
#!/usr/bin/php -Cq
|
||||
<?php
|
||||
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
|
||||
ini_set('memory_limit', '800M');
|
||||
ini_set('display_errors', 'stderr');
|
||||
ini_set('memory_limit', '800M');
|
||||
ini_set('display_errors', 'stderr');
|
||||
|
||||
$aCMDOptions = array(
|
||||
"Import country language data from osm wiki",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
|
||||
$aCMDOptions = array(
|
||||
"Import country language data from osm wiki",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
|
||||
|
||||
include(CONST_InstallPath.'/settings/phrase_settings.php');
|
||||
include(CONST_InstallPath.'/settings/phrase_settings.php');
|
||||
|
||||
if (true)
|
||||
{
|
||||
$sURL = 'http://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Country_Codes';
|
||||
$sWikiPageXML = file_get_contents($sURL);
|
||||
if (preg_match_all('#\\| ([a-z]{2}) \\|\\| [^|]+\\|\\| ([a-z,]+)#', $sWikiPageXML, $aMatches, PREG_SET_ORDER))
|
||||
{
|
||||
foreach($aMatches as $aMatch)
|
||||
{
|
||||
$aLanguages = explode(',', $aMatch[2]);
|
||||
foreach($aLanguages as $i => $s)
|
||||
{
|
||||
$aLanguages[$i] = '"'.pg_escape_string($s).'"';
|
||||
}
|
||||
echo "UPDATE country_name set country_default_language_codes = '{".join(',',$aLanguages)."}' where country_code = '".pg_escape_string($aMatch[1])."';\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
if (true)
|
||||
{
|
||||
$sURL = 'http://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Country_Codes';
|
||||
$sWikiPageXML = file_get_contents($sURL);
|
||||
if (preg_match_all('#\\| ([a-z]{2}) \\|\\| [^|]+\\|\\| ([a-z,]+)#', $sWikiPageXML, $aMatches, PREG_SET_ORDER))
|
||||
{
|
||||
foreach($aMatches as $aMatch)
|
||||
{
|
||||
$aLanguages = explode(',', $aMatch[2]);
|
||||
foreach($aLanguages as $i => $s)
|
||||
{
|
||||
$aLanguages[$i] = '"'.pg_escape_string($s).'"';
|
||||
}
|
||||
echo "UPDATE country_name set country_default_language_codes = '{".join(',',$aLanguages)."}' where country_code = '".pg_escape_string($aMatch[1])."';\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,32 +11,32 @@ echo "CREATE TABLE wikipedia_redirect (language text, from_title text, to_title
|
||||
|
||||
for i in "${language[@]}"
|
||||
do
|
||||
wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-page.sql.gz
|
||||
wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-pagelinks.sql.gz
|
||||
wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-langlinks.sql.gz
|
||||
wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-redirect.sql.gz
|
||||
wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-page.sql.gz
|
||||
wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-pagelinks.sql.gz
|
||||
wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-langlinks.sql.gz
|
||||
wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-redirect.sql.gz
|
||||
done
|
||||
|
||||
for i in "${language[@]}"
|
||||
do
|
||||
gzip -dc ${i}wiki-latest-pagelinks.sql.gz | sed "s/\`pagelinks\`/\`${i}pagelinks\`/g" | $mysql2pgsqlcmd | $psqlcmd
|
||||
gzip -dc ${i}wiki-latest-page.sql.gz | sed "s/\`page\`/\`${i}page\`/g" | $mysql2pgsqlcmd | $psqlcmd
|
||||
gzip -dc ${i}wiki-latest-langlinks.sql.gz | sed "s/\`langlinks\`/\`${i}langlinks\`/g" | $mysql2pgsqlcmd | $psqlcmd
|
||||
gzip -dc ${i}wiki-latest-redirect.sql.gz | sed "s/\`redirect\`/\`${i}redirect\`/g" | $mysql2pgsqlcmd | $psqlcmd
|
||||
gzip -dc ${i}wiki-latest-pagelinks.sql.gz | sed "s/\`pagelinks\`/\`${i}pagelinks\`/g" | $mysql2pgsqlcmd | $psqlcmd
|
||||
gzip -dc ${i}wiki-latest-page.sql.gz | sed "s/\`page\`/\`${i}page\`/g" | $mysql2pgsqlcmd | $psqlcmd
|
||||
gzip -dc ${i}wiki-latest-langlinks.sql.gz | sed "s/\`langlinks\`/\`${i}langlinks\`/g" | $mysql2pgsqlcmd | $psqlcmd
|
||||
gzip -dc ${i}wiki-latest-redirect.sql.gz | sed "s/\`redirect\`/\`${i}redirect\`/g" | $mysql2pgsqlcmd | $psqlcmd
|
||||
done
|
||||
|
||||
for i in "${language[@]}"
|
||||
do
|
||||
echo "create table ${i}pagelinkcount as select pl_title as title,count(*) as count from ${i}pagelinks where pl_namespace = 0 group by pl_title;" | $psqlcmd
|
||||
echo "insert into linkcounts select '${i}',pl_title,count(*) from ${i}pagelinks where pl_namespace = 0 group by pl_title;" | $psqlcmd
|
||||
echo "insert into wikipedia_redirect select '${i}',page_title,rd_title from ${i}redirect join ${i}page on (rd_from = page_id) where page_namespace = 0 and rd_namespace = 0;" | $psqlcmd
|
||||
echo "alter table ${i}pagelinkcount add column othercount integer;" | $psqlcmd
|
||||
echo "update ${i}pagelinkcount set othercount = 0;" | $psqlcmd
|
||||
for j in "${language[@]}"
|
||||
do
|
||||
echo "update ${i}pagelinkcount set othercount = ${i}pagelinkcount.othercount + x.count from (select page_title as title,count from ${i}langlinks join ${i}page on (ll_from = page_id) join ${j}pagelinkcount on (ll_lang = '${j}' and ll_title = title)) as x where x.title = ${i}pagelinkcount.title;" | $psqlcmd
|
||||
done
|
||||
echo "insert into wikipedia_article select '${i}', title, count, othercount, count+othercount from ${i}pagelinkcount;" | $psqlcmd
|
||||
echo "create table ${i}pagelinkcount as select pl_title as title,count(*) as count from ${i}pagelinks where pl_namespace = 0 group by pl_title;" | $psqlcmd
|
||||
echo "insert into linkcounts select '${i}',pl_title,count(*) from ${i}pagelinks where pl_namespace = 0 group by pl_title;" | $psqlcmd
|
||||
echo "insert into wikipedia_redirect select '${i}',page_title,rd_title from ${i}redirect join ${i}page on (rd_from = page_id) where page_namespace = 0 and rd_namespace = 0;" | $psqlcmd
|
||||
echo "alter table ${i}pagelinkcount add column othercount integer;" | $psqlcmd
|
||||
echo "update ${i}pagelinkcount set othercount = 0;" | $psqlcmd
|
||||
for j in "${language[@]}"
|
||||
do
|
||||
echo "update ${i}pagelinkcount set othercount = ${i}pagelinkcount.othercount + x.count from (select page_title as title,count from ${i}langlinks join ${i}page on (ll_from = page_id) join ${j}pagelinkcount on (ll_lang = '${j}' and ll_title = title)) as x where x.title = ${i}pagelinkcount.title;" | $psqlcmd
|
||||
done
|
||||
echo "insert into wikipedia_article select '${i}', title, count, othercount, count+othercount from ${i}pagelinkcount;" | $psqlcmd
|
||||
done
|
||||
|
||||
echo "update wikipedia_article set importance = log(totalcount)/log((select max(totalcount) from wikipedia_article))" | $psqlcmd
|
||||
|
||||
@@ -1,61 +1,61 @@
|
||||
#!/usr/bin/php -Cq
|
||||
<?php
|
||||
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
|
||||
$aCMDOptions = array(
|
||||
"Create and setup nominatim search system",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
$aCMDOptions = array(
|
||||
"Create and setup nominatim search system",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
|
||||
array('parse-tiger', '', 0, 1, 1, 1, 'realpath', 'Convert tiger edge files to nominatim sql import - datafiles from 2011 or later (source: edges directory of tiger data)'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
|
||||
array('parse-tiger', '', 0, 1, 1, 1, 'realpath', 'Convert tiger edge files to nominatim sql import - datafiles from 2011 or later (source: edges directory of tiger data)'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
|
||||
|
||||
|
||||
if (isset($aCMDResult['parse-tiger']))
|
||||
{
|
||||
if (!file_exists(CONST_Tiger_Data_Path)) mkdir(CONST_Tiger_Data_Path);
|
||||
if (isset($aCMDResult['parse-tiger']))
|
||||
{
|
||||
if (!file_exists(CONST_Tiger_Data_Path)) mkdir(CONST_Tiger_Data_Path);
|
||||
|
||||
$sTempDir = tempnam('/tmp', 'tiger');
|
||||
unlink($sTempDir);
|
||||
mkdir($sTempDir);
|
||||
$sTempDir = tempnam('/tmp', 'tiger');
|
||||
unlink($sTempDir);
|
||||
mkdir($sTempDir);
|
||||
|
||||
foreach(glob($aCMDResult['parse-tiger'].'/tl_20??_?????_edges.zip', 0) as $sImportFile)
|
||||
{
|
||||
set_time_limit(30);
|
||||
preg_match('#([0-9]{5})_(.*)#',basename($sImportFile), $aMatch);
|
||||
$sCountyID = $aMatch[1];
|
||||
echo "Processing ".$sCountyID."...\n";
|
||||
$sUnzipCmd = "unzip -d $sTempDir $sImportFile";
|
||||
exec($sUnzipCmd);
|
||||
$sShapeFile = $sTempDir.'/'.basename($sImportFile, '.zip').'.shp';
|
||||
if (!file_exists($sShapeFile))
|
||||
{
|
||||
echo "Failed unzip ($sImportFile)\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
$sParseCmd = CONST_BasePath.'/utils/tigerAddressImport.py '.$sShapeFile;
|
||||
exec($sParseCmd);
|
||||
$sOsmFile = $sTempDir.'/'.basename($sImportFile, '.zip').'.osm1.osm';
|
||||
if (!file_exists($sOsmFile))
|
||||
{
|
||||
echo "Failed parse ($sImportFile)\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
copy($sOsmFile, CONST_Tiger_Data_Path.'/'.$sCountyID.'.sql');
|
||||
}
|
||||
}
|
||||
// Cleanup
|
||||
foreach(glob($sTempDir.'/*') as $sTmpFile)
|
||||
{
|
||||
unlink($sTmpFile);
|
||||
}
|
||||
foreach(glob($aCMDResult['parse-tiger'].'/tl_20??_?????_edges.zip', 0) as $sImportFile)
|
||||
{
|
||||
set_time_limit(30);
|
||||
preg_match('#([0-9]{5})_(.*)#',basename($sImportFile), $aMatch);
|
||||
$sCountyID = $aMatch[1];
|
||||
echo "Processing ".$sCountyID."...\n";
|
||||
$sUnzipCmd = "unzip -d $sTempDir $sImportFile";
|
||||
exec($sUnzipCmd);
|
||||
$sShapeFile = $sTempDir.'/'.basename($sImportFile, '.zip').'.shp';
|
||||
if (!file_exists($sShapeFile))
|
||||
{
|
||||
echo "Failed unzip ($sImportFile)\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
$sParseCmd = CONST_BasePath.'/utils/tigerAddressImport.py '.$sShapeFile;
|
||||
exec($sParseCmd);
|
||||
$sOsmFile = $sTempDir.'/'.basename($sImportFile, '.zip').'.osm1.osm';
|
||||
if (!file_exists($sOsmFile))
|
||||
{
|
||||
echo "Failed parse ($sImportFile)\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
copy($sOsmFile, CONST_Tiger_Data_Path.'/'.$sCountyID.'.sql');
|
||||
}
|
||||
}
|
||||
// Cleanup
|
||||
foreach(glob($sTempDir.'/*') as $sTmpFile)
|
||||
{
|
||||
unlink($sTmpFile);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,55 +1,52 @@
|
||||
#!/usr/bin/php -Cq
|
||||
<?php
|
||||
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
require_once(CONST_BasePath.'/lib/Geocode.php');
|
||||
require_once(CONST_BasePath.'/lib/ParameterParser.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
require_once(CONST_BasePath.'/lib/Geocode.php');
|
||||
require_once(CONST_BasePath.'/lib/ParameterParser.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
|
||||
$aCMDOptions = array(
|
||||
"Query database from command line. Returns search result as JSON.",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
$aCMDOptions = array(
|
||||
"Query database from command line. Returns search result as JSON.",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
|
||||
array('search', '', 0, 1, 1, 1, 'string', 'Search for given term or coordinate'),
|
||||
array('search', '', 0, 1, 1, 1, 'string', 'Search for given term or coordinate'),
|
||||
|
||||
array('accept-language', '', 0, 1, 1, 1, 'string', 'Preferred language order for showing search results'),
|
||||
array('bounded', '', 0, 1, 0, 0, 'bool', 'Restrict results to given viewbox'),
|
||||
array('nodedupe', '', 0, 1, 0, 0, 'bool', 'Do not remove duplicate results'),
|
||||
array('limit', '', 0, 1, 1, 1, 'int', 'Maximum number of results returned (default: 10)'),
|
||||
array('exclude_place_ids', '', 0, 1, 1, 1, 'string', 'Comma-separated list of place ids to exclude from results'),
|
||||
array('featureType', '', 0, 1, 1, 1, 'string', 'Restrict results to certain features (country, state,city,settlement)'),
|
||||
array('countrycodes', '', 0, 1, 1, 1, 'string', 'Comma-separated list of countries to restrict search to'),
|
||||
array('viewbox', '', 0, 1, 1, 1, 'string', 'Prefer results in given view box')
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
|
||||
array('accept-language', '', 0, 1, 1, 1, 'string', 'Preferred language order for showing search results'),
|
||||
array('bounded', '', 0, 1, 0, 0, 'bool', 'Restrict results to given viewbox'),
|
||||
array('nodedupe', '', 0, 1, 0, 0, 'bool', 'Do not remove duplicate results'),
|
||||
array('limit', '', 0, 1, 1, 1, 'int', 'Maximum number of results returned (default: 10)'),
|
||||
array('exclude_place_ids', '', 0, 1, 1, 1, 'string', 'Comma-separated list of place ids to exclude from results'),
|
||||
array('featureType', '', 0, 1, 1, 1, 'string', 'Restrict results to certain features (country, state,city,settlement)'),
|
||||
array('countrycodes', '', 0, 1, 1, 1, 'string', 'Comma-separated list of countries to restrict search to'),
|
||||
array('viewbox', '', 0, 1, 1, 1, 'string', 'Prefer results in given view box')
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
|
||||
|
||||
$oDB =& getDB();
|
||||
$oParams = new ParameterParser($aCMDResult);
|
||||
$oDB =& getDB();
|
||||
$oParams = new ParameterParser($aCMDResult);
|
||||
|
||||
if ($oParams->getBool('search'))
|
||||
{
|
||||
if (isset($aCMDResult['nodedupe'])) $aCMDResult['dedupe'] = 'false';
|
||||
if ($oParams->getBool('search'))
|
||||
{
|
||||
if (isset($aCMDResult['nodedupe'])) $aCMDResult['dedupe'] = 'false';
|
||||
|
||||
$oGeocode = new Geocode($oDB);
|
||||
|
||||
$oGeocode->setLanguagePreference($oParams->getPreferredLanguages(false));
|
||||
$oGeocode->loadParamArray($oParams);
|
||||
$oGeocode->setQuery($aCMDResult['search']);
|
||||
|
||||
$aSearchResults = $oGeocode->lookup();
|
||||
|
||||
if (version_compare(phpversion(), "5.4.0", '<'))
|
||||
echo json_encode($aSearchResults);
|
||||
else
|
||||
echo json_encode($aSearchResults, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)."\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
showUsage($aCMDOptions, true);
|
||||
}
|
||||
$oGeocode = new Geocode($oDB);
|
||||
|
||||
$oGeocode->setLanguagePreference($oParams->getPreferredLanguages(false));
|
||||
$oGeocode->loadParamArray($oParams);
|
||||
$oGeocode->setQuery($aCMDResult['search']);
|
||||
|
||||
$aSearchResults = $oGeocode->lookup();
|
||||
|
||||
if (version_compare(phpversion(), "5.4.0", '<'))
|
||||
echo json_encode($aSearchResults);
|
||||
else
|
||||
echo json_encode($aSearchResults, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)."\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
showUsage($aCMDOptions, true);
|
||||
}
|
||||
|
||||
@@ -1,78 +1,78 @@
|
||||
#!/usr/bin/php -Cq
|
||||
<?php
|
||||
|
||||
// Apache log file
|
||||
$sFile = "sample.log.txt";
|
||||
$sHost1 = 'http://mq-open-search-lm02.ihost.aol.com:8000/nominatim/v1';
|
||||
$sHost2 = 'http://mq-open-search-lm03.ihost.aol.com:8000/nominatim/v1';
|
||||
// Apache log file
|
||||
$sFile = "sample.log.txt";
|
||||
$sHost1 = 'http://mq-open-search-lm02.ihost.aol.com:8000/nominatim/v1';
|
||||
$sHost2 = 'http://mq-open-search-lm03.ihost.aol.com:8000/nominatim/v1';
|
||||
|
||||
|
||||
$sHost1Escaped = str_replace('/', '\\/', $sHost1);
|
||||
$sHost2Escaped = str_replace('/', '\\/', $sHost2);
|
||||
$sHost1Escaped = str_replace('/', '\\/', $sHost1);
|
||||
$sHost2Escaped = str_replace('/', '\\/', $sHost2);
|
||||
|
||||
$aToDo = array(251, 293, 328, 399.1, 455.1, 479, 496, 499, 574, 609, 702, 790, 846, 865, 878, 894, 902, 961, 980);
|
||||
$aToDo = array(251, 293, 328, 399.1, 455.1, 479, 496, 499, 574, 609, 702, 790, 846, 865, 878, 894, 902, 961, 980);
|
||||
|
||||
$hFile = @fopen($sFile, "r");
|
||||
if (!$hFile)
|
||||
{
|
||||
echo "Unable to open file: $sFile\n";
|
||||
exit;
|
||||
}
|
||||
$hFile = @fopen($sFile, "r");
|
||||
if (!$hFile)
|
||||
{
|
||||
echo "Unable to open file: $sFile\n";
|
||||
exit;
|
||||
}
|
||||
|
||||
$i = 0;
|
||||
while (($sLine = fgets($hFile, 10000)) !== false)
|
||||
{
|
||||
$i++;
|
||||
if (!in_array($i, $aToDo)) continue;
|
||||
$i = 0;
|
||||
while (($sLine = fgets($hFile, 10000)) !== false)
|
||||
{
|
||||
$i++;
|
||||
if (!in_array($i, $aToDo)) continue;
|
||||
|
||||
if (preg_match('#"GET (.*) HTTP/1.[01]"#', $sLine, $aResult))
|
||||
{
|
||||
$sURL1 = $sHost1.$aResult[1];
|
||||
$sURL2 = $sHost2.$aResult[1];
|
||||
if (preg_match('#"GET (.*) HTTP/1.[01]"#', $sLine, $aResult))
|
||||
{
|
||||
$sURL1 = $sHost1.$aResult[1];
|
||||
$sURL2 = $sHost2.$aResult[1];
|
||||
|
||||
$sRes1 = '';
|
||||
$k = 0;
|
||||
while(!$sRes1 && $k < 10)
|
||||
{
|
||||
$sRes1 = file_get_contents($sURL1);
|
||||
$k++;
|
||||
if (!$sRes1) sleep(10);
|
||||
}
|
||||
$sRes2 = file_get_contents($sURL2);
|
||||
$sRes1 = '';
|
||||
$k = 0;
|
||||
while(!$sRes1 && $k < 10)
|
||||
{
|
||||
$sRes1 = file_get_contents($sURL1);
|
||||
$k++;
|
||||
if (!$sRes1) sleep(10);
|
||||
}
|
||||
$sRes2 = file_get_contents($sURL2);
|
||||
|
||||
// Strip out the things that will always change
|
||||
$sRes1 = preg_replace('# timestamp=\'[^\']*\'#', '', $sRes1);
|
||||
$sRes1 = str_replace($sHost1, '', $sRes1);
|
||||
$sRes1 = str_replace($sHost1Escaped, '', $sRes1);
|
||||
$sRes2 = preg_replace('# timestamp=\'[^\']*\'#', '', $sRes2);
|
||||
$sRes2 = str_replace($sHost2, '', $sRes2);
|
||||
$sRes2 = str_replace($sHost2Escaped, '', $sRes2);
|
||||
// Strip out the things that will always change
|
||||
$sRes1 = preg_replace('# timestamp=\'[^\']*\'#', '', $sRes1);
|
||||
$sRes1 = str_replace($sHost1, '', $sRes1);
|
||||
$sRes1 = str_replace($sHost1Escaped, '', $sRes1);
|
||||
$sRes2 = preg_replace('# timestamp=\'[^\']*\'#', '', $sRes2);
|
||||
$sRes2 = str_replace($sHost2, '', $sRes2);
|
||||
$sRes2 = str_replace($sHost2Escaped, '', $sRes2);
|
||||
|
||||
if ($sRes1 != $sRes2)
|
||||
{
|
||||
echo "$i:\n";
|
||||
var_dump($sURL1, $sURL2);
|
||||
if ($sRes1 != $sRes2)
|
||||
{
|
||||
echo "$i:\n";
|
||||
var_dump($sURL1, $sURL2);
|
||||
|
||||
$sRes = $sURL1.":\n";
|
||||
for ($j = 0; $j < strlen($sRes1); $j+=40)
|
||||
{
|
||||
$sRes .= substr($sRes1, $j, 40)."\n";
|
||||
}
|
||||
file_put_contents('log/'.$i.'.1', $sRes);
|
||||
$sRes = $sURL1.":\n";
|
||||
for ($j = 0; $j < strlen($sRes1); $j+=40)
|
||||
{
|
||||
$sRes .= substr($sRes1, $j, 40)."\n";
|
||||
}
|
||||
file_put_contents('log/'.$i.'.1', $sRes);
|
||||
|
||||
$sRes = $sURL2.":\n";
|
||||
for ($j = 0; $j < strlen($sRes2); $j+=40)
|
||||
{
|
||||
$sRes .= substr($sRes2, $j, 40)."\n";
|
||||
}
|
||||
file_put_contents('log/'.$i.'.2', $sRes);
|
||||
}
|
||||
echo ".\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
var_dump($sLine);
|
||||
}
|
||||
}
|
||||
$sRes = $sURL2.":\n";
|
||||
for ($j = 0; $j < strlen($sRes2); $j+=40)
|
||||
{
|
||||
$sRes .= substr($sRes2, $j, 40)."\n";
|
||||
}
|
||||
file_put_contents('log/'.$i.'.2', $sRes);
|
||||
}
|
||||
echo ".\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
var_dump($sLine);
|
||||
}
|
||||
}
|
||||
|
||||
fclose($hFile);
|
||||
fclose($hFile);
|
||||
|
||||
1764
utils/setup.php
1764
utils/setup.php
File diff suppressed because it is too large
Load Diff
@@ -1,114 +1,114 @@
|
||||
#!/usr/bin/php -Cq
|
||||
<?php
|
||||
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
ini_set('display_errors', 'stderr');
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
ini_set('display_errors', 'stderr');
|
||||
|
||||
$aCMDOptions = array(
|
||||
"Import and export special phrases",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
array('countries', '', 0, 1, 0, 0, 'bool', 'Create import script for country codes and names'),
|
||||
array('wiki-import', '', 0, 1, 0, 0, 'bool', 'Create import script for search phrases '),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
|
||||
$aCMDOptions = array(
|
||||
"Import and export special phrases",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
array('countries', '', 0, 1, 0, 0, 'bool', 'Create import script for country codes and names'),
|
||||
array('wiki-import', '', 0, 1, 0, 0, 'bool', 'Create import script for search phrases '),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
|
||||
|
||||
include(CONST_InstallPath.'/settings/phrase_settings.php');
|
||||
include(CONST_InstallPath.'/settings/phrase_settings.php');
|
||||
|
||||
|
||||
if ($aCMDResult['countries']) {
|
||||
echo "select getorcreate_country(make_standard_name('uk'), 'gb');\n";
|
||||
echo "select getorcreate_country(make_standard_name('united states'), 'us');\n";
|
||||
echo "select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x;\n";
|
||||
if ($aCMDResult['countries']) {
|
||||
echo "select getorcreate_country(make_standard_name('uk'), 'gb');\n";
|
||||
echo "select getorcreate_country(make_standard_name('united states'), 'us');\n";
|
||||
echo "select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x;\n";
|
||||
|
||||
echo "select count(*) from (select getorcreate_country(make_standard_name(get_name_by_language(country_name.name,ARRAY['name'])), country_code) from country_name where get_name_by_language(country_name.name, ARRAY['name']) is not null) as x;\n";
|
||||
foreach($aLanguageIn as $sLanguage)
|
||||
{
|
||||
echo "select count(*) from (select getorcreate_country(make_standard_name(get_name_by_language(country_name.name,ARRAY['name:".$sLanguage."'])), country_code) from country_name where get_name_by_language(country_name.name, ARRAY['name:".$sLanguage."']) is not null) as x;\n";
|
||||
echo "select count(*) from (select getorcreate_country(make_standard_name(get_name_by_language(country_name.name,ARRAY['name'])), country_code) from country_name where get_name_by_language(country_name.name, ARRAY['name']) is not null) as x;\n";
|
||||
foreach($aLanguageIn as $sLanguage)
|
||||
{
|
||||
echo "select count(*) from (select getorcreate_country(make_standard_name(get_name_by_language(country_name.name,ARRAY['name:".$sLanguage."'])), country_code) from country_name where get_name_by_language(country_name.name, ARRAY['name:".$sLanguage."']) is not null) as x;\n";
|
||||
}
|
||||
}
|
||||
|
||||
if ($aCMDResult['wiki-import'])
|
||||
{
|
||||
$aPairs = array();
|
||||
|
||||
foreach($aLanguageIn as $sLanguage)
|
||||
{
|
||||
$sURL = 'http://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/'.strtoupper($sLanguage);
|
||||
$sWikiPageXML = file_get_contents($sURL);
|
||||
if (preg_match_all('#\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([\\-YN])#', $sWikiPageXML, $aMatches, PREG_SET_ORDER))
|
||||
{
|
||||
foreach($aMatches as $aMatch)
|
||||
{
|
||||
$sLabel = trim($aMatch[1]);
|
||||
$sClass = trim($aMatch[2]);
|
||||
$sType = trim($aMatch[3]);
|
||||
# hack around a bug where building=yes was imported with
|
||||
# quotes into the wiki
|
||||
$sType = preg_replace('/"/', '', $sType);
|
||||
# sanity check, in case somebody added garbage in the wiki
|
||||
if (preg_match('/^\\w+$/', $sClass) < 1 ||
|
||||
preg_match('/^\\w+$/', $sType) < 1) {
|
||||
trigger_error("Bad class/type for language $sLanguage: $sClass=$sType");
|
||||
exit;
|
||||
}
|
||||
# blacklisting: disallow certain class/type combinations
|
||||
if (isset($aTagsBlacklist[$sClass]) && in_array($sType, $aTagsBlacklist[$sClass])) {
|
||||
# fwrite(STDERR, "Blacklisted: ".$sClass."/".$sType."\n");
|
||||
continue;
|
||||
}
|
||||
# whitelisting: if class is in whitelist, allow only tags in the list
|
||||
if (isset($aTagsWhitelist[$sClass]) && !in_array($sType, $aTagsWhitelist[$sClass])) {
|
||||
# fwrite(STDERR, "Non-Whitelisted: ".$sClass."/".$sType."\n");
|
||||
continue;
|
||||
}
|
||||
$aPairs[$sClass.'|'.$sType] = array($sClass, $sType);
|
||||
|
||||
switch(trim($aMatch[4]))
|
||||
{
|
||||
case 'near':
|
||||
echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType', 'near');\n";
|
||||
break;
|
||||
case 'in':
|
||||
echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType', 'in');\n";
|
||||
break;
|
||||
default:
|
||||
echo "select getorcreate_amenity(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType');\n";
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($aCMDResult['wiki-import'])
|
||||
{
|
||||
$aPairs = array();
|
||||
echo "create index idx_placex_classtype on placex (class, type);";
|
||||
|
||||
foreach($aLanguageIn as $sLanguage)
|
||||
{
|
||||
$sURL = 'http://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/'.strtoupper($sLanguage);
|
||||
$sWikiPageXML = file_get_contents($sURL);
|
||||
if (preg_match_all('#\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([\\-YN])#', $sWikiPageXML, $aMatches, PREG_SET_ORDER))
|
||||
{
|
||||
foreach($aMatches as $aMatch)
|
||||
{
|
||||
$sLabel = trim($aMatch[1]);
|
||||
$sClass = trim($aMatch[2]);
|
||||
$sType = trim($aMatch[3]);
|
||||
# hack around a bug where building=yes was imported with
|
||||
# quotes into the wiki
|
||||
$sType = preg_replace('/"/', '', $sType);
|
||||
# sanity check, in case somebody added garbage in the wiki
|
||||
if (preg_match('/^\\w+$/', $sClass) < 1 ||
|
||||
preg_match('/^\\w+$/', $sType) < 1) {
|
||||
trigger_error("Bad class/type for language $sLanguage: $sClass=$sType");
|
||||
exit;
|
||||
}
|
||||
# blacklisting: disallow certain class/type combinations
|
||||
if (isset($aTagsBlacklist[$sClass]) && in_array($sType, $aTagsBlacklist[$sClass])) {
|
||||
# fwrite(STDERR, "Blacklisted: ".$sClass."/".$sType."\n");
|
||||
continue;
|
||||
}
|
||||
# whitelisting: if class is in whitelist, allow only tags in the list
|
||||
if (isset($aTagsWhitelist[$sClass]) && !in_array($sType, $aTagsWhitelist[$sClass])) {
|
||||
# fwrite(STDERR, "Non-Whitelisted: ".$sClass."/".$sType."\n");
|
||||
continue;
|
||||
}
|
||||
$aPairs[$sClass.'|'.$sType] = array($sClass, $sType);
|
||||
foreach($aPairs as $aPair)
|
||||
{
|
||||
echo "create table place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1]);
|
||||
if (CONST_Tablespace_Aux_Data)
|
||||
echo " tablespace ".CONST_Tablespace_Aux_Data;
|
||||
echo " as select place_id as place_id,st_centroid(geometry) as centroid from placex where ";
|
||||
echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."'";
|
||||
echo ";\n";
|
||||
|
||||
switch(trim($aMatch[4]))
|
||||
{
|
||||
case 'near':
|
||||
echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType', 'near');\n";
|
||||
break;
|
||||
case 'in':
|
||||
echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType', 'in');\n";
|
||||
break;
|
||||
default:
|
||||
echo "select getorcreate_amenity(make_standard_name('".pg_escape_string($sLabel)."'), '$sClass', '$sType');\n";
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_centroid ";
|
||||
echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING GIST (centroid)";
|
||||
if (CONST_Tablespace_Aux_Index)
|
||||
echo " tablespace ".CONST_Tablespace_Aux_Index;
|
||||
echo ";\n";
|
||||
|
||||
echo "create index idx_placex_classtype on placex (class, type);";
|
||||
echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_place_id ";
|
||||
echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING btree(place_id)";
|
||||
if (CONST_Tablespace_Aux_Index)
|
||||
echo " tablespace ".CONST_Tablespace_Aux_Index;
|
||||
echo ";\n";
|
||||
|
||||
foreach($aPairs as $aPair)
|
||||
{
|
||||
echo "create table place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1]);
|
||||
if (CONST_Tablespace_Aux_Data)
|
||||
echo " tablespace ".CONST_Tablespace_Aux_Data;
|
||||
echo " as select place_id as place_id,st_centroid(geometry) as centroid from placex where ";
|
||||
echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."'";
|
||||
echo ";\n";
|
||||
echo "GRANT SELECT ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1]).' TO "'.CONST_Database_Web_User."\";\n";
|
||||
|
||||
echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_centroid ";
|
||||
echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING GIST (centroid)";
|
||||
if (CONST_Tablespace_Aux_Index)
|
||||
echo " tablespace ".CONST_Tablespace_Aux_Index;
|
||||
echo ";\n";
|
||||
}
|
||||
|
||||
echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_place_id ";
|
||||
echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING btree(place_id)";
|
||||
if (CONST_Tablespace_Aux_Index)
|
||||
echo " tablespace ".CONST_Tablespace_Aux_Index;
|
||||
echo ";\n";
|
||||
|
||||
echo "GRANT SELECT ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1]).' TO "'.CONST_Database_Web_User."\";\n";
|
||||
|
||||
}
|
||||
|
||||
echo "drop index idx_placex_classtype;";
|
||||
}
|
||||
echo "drop index idx_placex_classtype;";
|
||||
}
|
||||
|
||||
648
utils/update.php
648
utils/update.php
@@ -1,376 +1,376 @@
|
||||
#!/usr/bin/php -Cq
|
||||
<?php
|
||||
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
|
||||
$aCMDOptions = array(
|
||||
"Import / update / index osm data",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
$aCMDOptions = array(
|
||||
"Import / update / index osm data",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
|
||||
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
|
||||
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
|
||||
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),
|
||||
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
|
||||
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
|
||||
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
|
||||
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),
|
||||
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
|
||||
|
||||
array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
|
||||
array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
|
||||
|
||||
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
|
||||
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
|
||||
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
|
||||
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
|
||||
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
|
||||
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
|
||||
|
||||
array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
|
||||
array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
|
||||
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
|
||||
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
|
||||
array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
|
||||
array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
|
||||
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
|
||||
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
|
||||
|
||||
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
|
||||
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
|
||||
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
|
||||
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
|
||||
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
|
||||
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
|
||||
|
||||
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
|
||||
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
|
||||
|
||||
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
|
||||
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
|
||||
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
|
||||
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
|
||||
|
||||
date_default_timezone_set('Etc/UTC');
|
||||
date_default_timezone_set('Etc/UTC');
|
||||
|
||||
$oDB =& getDB();
|
||||
$oDB =& getDB();
|
||||
|
||||
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
|
||||
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
|
||||
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
|
||||
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
|
||||
|
||||
// cache memory to be used by osm2pgsql, should not be more than the available memory
|
||||
$iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
|
||||
if ($iCacheMemory + 500 > getTotalMemoryMB())
|
||||
{
|
||||
$iCacheMemory = getCacheMemoryMB();
|
||||
echo "WARNING: resetting cache memory to $iCacheMemory\n";
|
||||
}
|
||||
$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
|
||||
if (!is_null(CONST_Osm2pgsql_Flatnode_File))
|
||||
{
|
||||
$sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
|
||||
}
|
||||
// cache memory to be used by osm2pgsql, should not be more than the available memory
|
||||
$iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
|
||||
if ($iCacheMemory + 500 > getTotalMemoryMB())
|
||||
{
|
||||
$iCacheMemory = getCacheMemoryMB();
|
||||
echo "WARNING: resetting cache memory to $iCacheMemory\n";
|
||||
}
|
||||
$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
|
||||
if (!is_null(CONST_Osm2pgsql_Flatnode_File))
|
||||
{
|
||||
$sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
|
||||
}
|
||||
|
||||
|
||||
if (isset($aResult['import-diff']))
|
||||
{
|
||||
// import diff directly (e.g. from osmosis --rri)
|
||||
$sNextFile = $aResult['import-diff'];
|
||||
if (!file_exists($sNextFile))
|
||||
{
|
||||
fail("Cannot open $sNextFile\n");
|
||||
}
|
||||
if (isset($aResult['import-diff']))
|
||||
{
|
||||
// import diff directly (e.g. from osmosis --rri)
|
||||
$sNextFile = $aResult['import-diff'];
|
||||
if (!file_exists($sNextFile))
|
||||
{
|
||||
fail("Cannot open $sNextFile\n");
|
||||
}
|
||||
|
||||
// Import the file
|
||||
$sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
|
||||
echo $sCMD."\n";
|
||||
exec($sCMD, $sJunk, $iErrorLevel);
|
||||
// Import the file
|
||||
$sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
|
||||
echo $sCMD."\n";
|
||||
exec($sCMD, $sJunk, $iErrorLevel);
|
||||
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
fail("Error from osm2pgsql, $iErrorLevel\n");
|
||||
}
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
fail("Error from osm2pgsql, $iErrorLevel\n");
|
||||
}
|
||||
|
||||
// Don't update the import status - we don't know what this file contains
|
||||
}
|
||||
// Don't update the import status - we don't know what this file contains
|
||||
}
|
||||
|
||||
$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
|
||||
$bHaveDiff = false;
|
||||
if (isset($aResult['import-file']) && $aResult['import-file'])
|
||||
{
|
||||
$bHaveDiff = true;
|
||||
$sCMD = CONST_Osmosis_Binary.' --read-xml \''.$aResult['import-file'].'\' --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
|
||||
echo $sCMD."\n";
|
||||
exec($sCMD, $sJunk, $iErrorLevel);
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n");
|
||||
}
|
||||
}
|
||||
$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
|
||||
$bHaveDiff = false;
|
||||
if (isset($aResult['import-file']) && $aResult['import-file'])
|
||||
{
|
||||
$bHaveDiff = true;
|
||||
$sCMD = CONST_Osmosis_Binary.' --read-xml \''.$aResult['import-file'].'\' --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
|
||||
echo $sCMD."\n";
|
||||
exec($sCMD, $sJunk, $iErrorLevel);
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n");
|
||||
}
|
||||
}
|
||||
|
||||
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
|
||||
$sContentURL = '';
|
||||
if (isset($aResult['import-node']) && $aResult['import-node'])
|
||||
{
|
||||
if ($bUseOSMApi)
|
||||
{
|
||||
$sContentURL = 'http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
|
||||
}
|
||||
else
|
||||
{
|
||||
$sContentURL = 'http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
|
||||
}
|
||||
}
|
||||
if (isset($aResult['import-way']) && $aResult['import-way'])
|
||||
{
|
||||
if ($bUseOSMApi)
|
||||
{
|
||||
$sContentURL = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
|
||||
}
|
||||
else
|
||||
{
|
||||
$sContentURL = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
|
||||
}
|
||||
}
|
||||
if (isset($aResult['import-relation']) && $aResult['import-relation'])
|
||||
{
|
||||
if ($bUseOSMApi)
|
||||
{
|
||||
$sContentURLsModifyXMLstr = 'http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
|
||||
}
|
||||
else
|
||||
{
|
||||
$sContentURL = 'http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
|
||||
}
|
||||
}
|
||||
if ($sContentURL)
|
||||
{
|
||||
$sModifyXMLstr = file_get_contents($sContentURL);
|
||||
$bHaveDiff = true;
|
||||
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
|
||||
$sContentURL = '';
|
||||
if (isset($aResult['import-node']) && $aResult['import-node'])
|
||||
{
|
||||
if ($bUseOSMApi)
|
||||
{
|
||||
$sContentURL = 'http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
|
||||
}
|
||||
else
|
||||
{
|
||||
$sContentURL = 'http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
|
||||
}
|
||||
}
|
||||
if (isset($aResult['import-way']) && $aResult['import-way'])
|
||||
{
|
||||
if ($bUseOSMApi)
|
||||
{
|
||||
$sContentURL = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
|
||||
}
|
||||
else
|
||||
{
|
||||
$sContentURL = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
|
||||
}
|
||||
}
|
||||
if (isset($aResult['import-relation']) && $aResult['import-relation'])
|
||||
{
|
||||
if ($bUseOSMApi)
|
||||
{
|
||||
$sContentURLsModifyXMLstr = 'http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
|
||||
}
|
||||
else
|
||||
{
|
||||
$sContentURL = 'http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
|
||||
}
|
||||
}
|
||||
if ($sContentURL)
|
||||
{
|
||||
$sModifyXMLstr = file_get_contents($sContentURL);
|
||||
$bHaveDiff = true;
|
||||
|
||||
$aSpec = array(
|
||||
0 => array("pipe", "r"), // stdin
|
||||
1 => array("pipe", "w"), // stdout
|
||||
2 => array("pipe", "w") // stderr
|
||||
);
|
||||
$sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
|
||||
echo $sCMD."\n";
|
||||
$hProc = proc_open($sCMD, $aSpec, $aPipes);
|
||||
if (!is_resource($hProc))
|
||||
{
|
||||
fail("Error converting osm to osc, osmosis failed\n");
|
||||
}
|
||||
fwrite($aPipes[0], $sModifyXMLstr);
|
||||
fclose($aPipes[0]);
|
||||
$sOut = stream_get_contents($aPipes[1]);
|
||||
if ($aResult['verbose']) echo $sOut;
|
||||
fclose($aPipes[1]);
|
||||
$sErrors = stream_get_contents($aPipes[2]);
|
||||
if ($aResult['verbose']) echo $sErrors;
|
||||
fclose($aPipes[2]);
|
||||
if ($iError = proc_close($hProc))
|
||||
{
|
||||
echo $sOut;
|
||||
echo $sErrors;
|
||||
fail("Error converting osm to osc, osmosis returned: $iError\n");
|
||||
}
|
||||
}
|
||||
$aSpec = array(
|
||||
0 => array("pipe", "r"), // stdin
|
||||
1 => array("pipe", "w"), // stdout
|
||||
2 => array("pipe", "w") // stderr
|
||||
);
|
||||
$sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
|
||||
echo $sCMD."\n";
|
||||
$hProc = proc_open($sCMD, $aSpec, $aPipes);
|
||||
if (!is_resource($hProc))
|
||||
{
|
||||
fail("Error converting osm to osc, osmosis failed\n");
|
||||
}
|
||||
fwrite($aPipes[0], $sModifyXMLstr);
|
||||
fclose($aPipes[0]);
|
||||
$sOut = stream_get_contents($aPipes[1]);
|
||||
if ($aResult['verbose']) echo $sOut;
|
||||
fclose($aPipes[1]);
|
||||
$sErrors = stream_get_contents($aPipes[2]);
|
||||
if ($aResult['verbose']) echo $sErrors;
|
||||
fclose($aPipes[2]);
|
||||
if ($iError = proc_close($hProc))
|
||||
{
|
||||
echo $sOut;
|
||||
echo $sErrors;
|
||||
fail("Error converting osm to osc, osmosis returned: $iError\n");
|
||||
}
|
||||
}
|
||||
|
||||
if ($bHaveDiff)
|
||||
{
|
||||
// import generated change file
|
||||
$sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
|
||||
echo $sCMD."\n";
|
||||
exec($sCMD, $sJunk, $iErrorLevel);
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
fail("osm2pgsql exited with error level $iErrorLevel\n");
|
||||
}
|
||||
}
|
||||
if ($bHaveDiff)
|
||||
{
|
||||
// import generated change file
|
||||
$sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
|
||||
echo $sCMD."\n";
|
||||
exec($sCMD, $sJunk, $iErrorLevel);
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
fail("osm2pgsql exited with error level $iErrorLevel\n");
|
||||
}
|
||||
}
|
||||
|
||||
if ($aResult['deduplicate'])
|
||||
{
|
||||
if ($aResult['deduplicate'])
|
||||
{
|
||||
|
||||
if (getPostgresVersion() < 9.3)
|
||||
{
|
||||
fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
|
||||
}
|
||||
if (getPostgresVersion() < 9.3)
|
||||
{
|
||||
fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
|
||||
}
|
||||
|
||||
$oDB =& getDB();
|
||||
$sSQL = 'select partition from country_name order by country_code';
|
||||
$aPartitions = chksql($oDB->getCol($sSQL));
|
||||
$aPartitions[] = 0;
|
||||
$oDB =& getDB();
|
||||
$sSQL = 'select partition from country_name order by country_code';
|
||||
$aPartitions = chksql($oDB->getCol($sSQL));
|
||||
$aPartitions[] = 0;
|
||||
|
||||
$sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
|
||||
$aDuplicateTokens = chksql($oDB->getAll($sSQL));
|
||||
foreach($aDuplicateTokens as $aToken)
|
||||
{
|
||||
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
|
||||
echo "Deduping ".$aToken['word_token']."\n";
|
||||
$sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc";
|
||||
$aTokenSet = chksql($oDB->getAll($sSQL));
|
||||
$sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
|
||||
$aDuplicateTokens = chksql($oDB->getAll($sSQL));
|
||||
foreach($aDuplicateTokens as $aToken)
|
||||
{
|
||||
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
|
||||
echo "Deduping ".$aToken['word_token']."\n";
|
||||
$sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc";
|
||||
$aTokenSet = chksql($oDB->getAll($sSQL));
|
||||
|
||||
$aKeep = array_shift($aTokenSet);
|
||||
$iKeepID = $aKeep['word_id'];
|
||||
$aKeep = array_shift($aTokenSet);
|
||||
$iKeepID = $aKeep['word_id'];
|
||||
|
||||
foreach($aTokenSet as $aRemove)
|
||||
{
|
||||
$sSQL = "update search_name set";
|
||||
$sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID."),";
|
||||
$sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
|
||||
$sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
|
||||
chksql($oDB->query($sSQL));
|
||||
foreach($aTokenSet as $aRemove)
|
||||
{
|
||||
$sSQL = "update search_name set";
|
||||
$sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID."),";
|
||||
$sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
|
||||
$sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
$sSQL = "update search_name set";
|
||||
$sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
|
||||
$sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
|
||||
chksql($oDB->query($sSQL));
|
||||
$sSQL = "update search_name set";
|
||||
$sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
|
||||
$sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
$sSQL = "update location_area_country set";
|
||||
$sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
|
||||
$sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
|
||||
chksql($oDB->query($sSQL));
|
||||
$sSQL = "update location_area_country set";
|
||||
$sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
|
||||
$sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
foreach ($aPartitions as $sPartition)
|
||||
{
|
||||
$sSQL = "update search_name_".$sPartition." set";
|
||||
$sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID.")";
|
||||
$sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
|
||||
chksql($oDB->query($sSQL));
|
||||
foreach ($aPartitions as $sPartition)
|
||||
{
|
||||
$sSQL = "update search_name_".$sPartition." set";
|
||||
$sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID.")";
|
||||
$sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
|
||||
chksql($oDB->query($sSQL));
|
||||
|
||||
$sSQL = "update location_area_country set";
|
||||
$sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
|
||||
$sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
|
||||
chksql($oDB->query($sSQL));
|
||||
}
|
||||
$sSQL = "update location_area_country set";
|
||||
$sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
|
||||
$sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
|
||||
chksql($oDB->query($sSQL));
|
||||
}
|
||||
|
||||
$sSQL = "delete from word where word_id = ".$aRemove['word_id'];
|
||||
chksql($oDB->query($sSQL));
|
||||
}
|
||||
}
|
||||
}
|
||||
$sSQL = "delete from word where word_id = ".$aRemove['word_id'];
|
||||
chksql($oDB->query($sSQL));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($aResult['index'])
|
||||
{
|
||||
passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
|
||||
}
|
||||
if ($aResult['index'])
|
||||
{
|
||||
passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
|
||||
}
|
||||
|
||||
if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
|
||||
{
|
||||
if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
|
||||
{
|
||||
|
||||
if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
|
||||
fail("Error: Update interval too low for download.geofabrik.de. Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n");
|
||||
}
|
||||
if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
|
||||
fail("Error: Update interval too low for download.geofabrik.de. Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n");
|
||||
}
|
||||
|
||||
$sImportFile = CONST_BasePath.'/data/osmosischange.osc';
|
||||
$sOsmosisConfigDirectory = CONST_InstallPath.'/settings';
|
||||
$sCMDDownload = CONST_Osmosis_Binary.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
|
||||
$sCMDCheckReplicationLag = CONST_Osmosis_Binary.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
|
||||
$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
|
||||
$sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
|
||||
$sImportFile = CONST_BasePath.'/data/osmosischange.osc';
|
||||
$sOsmosisConfigDirectory = CONST_InstallPath.'/settings';
|
||||
$sCMDDownload = CONST_Osmosis_Binary.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
|
||||
$sCMDCheckReplicationLag = CONST_Osmosis_Binary.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
|
||||
$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
|
||||
$sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
|
||||
|
||||
while(true)
|
||||
{
|
||||
$fStartTime = time();
|
||||
$iFileSize = 1001;
|
||||
while(true)
|
||||
{
|
||||
$fStartTime = time();
|
||||
$iFileSize = 1001;
|
||||
|
||||
if (!file_exists($sImportFile))
|
||||
{
|
||||
// First check if there are new updates published (except for minutelies - there's always new diffs to process)
|
||||
if ( CONST_Replication_Update_Interval > 60 )
|
||||
{
|
||||
if (!file_exists($sImportFile))
|
||||
{
|
||||
// First check if there are new updates published (except for minutelies - there's always new diffs to process)
|
||||
if ( CONST_Replication_Update_Interval > 60 )
|
||||
{
|
||||
|
||||
unset($aReplicationLag);
|
||||
exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
|
||||
while ($iErrorLevel > 0 || $aReplicationLag[0] < 1)
|
||||
{
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
echo "Error: $iErrorLevel. ";
|
||||
echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
echo ".";
|
||||
}
|
||||
sleep(CONST_Replication_Recheck_Interval);
|
||||
unset($aReplicationLag);
|
||||
exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
|
||||
}
|
||||
// There are new replication files - use osmosis to download the file
|
||||
echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
|
||||
}
|
||||
$fStartTime = time();
|
||||
$fCMDStartTime = time();
|
||||
echo $sCMDDownload."\n";
|
||||
exec($sCMDDownload, $sJunk, $iErrorLevel);
|
||||
while ($iErrorLevel > 0)
|
||||
{
|
||||
echo "Error: $iErrorLevel\n";
|
||||
sleep(60);
|
||||
echo 'Re-trying: '.$sCMDDownload."\n";
|
||||
exec($sCMDDownload, $sJunk, $iErrorLevel);
|
||||
}
|
||||
$iFileSize = filesize($sImportFile);
|
||||
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
||||
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
|
||||
var_Dump($sSQL);
|
||||
$oDB->query($sSQL);
|
||||
echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
||||
}
|
||||
unset($aReplicationLag);
|
||||
exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
|
||||
while ($iErrorLevel > 0 || $aReplicationLag[0] < 1)
|
||||
{
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
echo "Error: $iErrorLevel. ";
|
||||
echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
echo ".";
|
||||
}
|
||||
sleep(CONST_Replication_Recheck_Interval);
|
||||
unset($aReplicationLag);
|
||||
exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
|
||||
}
|
||||
// There are new replication files - use osmosis to download the file
|
||||
echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
|
||||
}
|
||||
$fStartTime = time();
|
||||
$fCMDStartTime = time();
|
||||
echo $sCMDDownload."\n";
|
||||
exec($sCMDDownload, $sJunk, $iErrorLevel);
|
||||
while ($iErrorLevel > 0)
|
||||
{
|
||||
echo "Error: $iErrorLevel\n";
|
||||
sleep(60);
|
||||
echo 'Re-trying: '.$sCMDDownload."\n";
|
||||
exec($sCMDDownload, $sJunk, $iErrorLevel);
|
||||
}
|
||||
$iFileSize = filesize($sImportFile);
|
||||
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
||||
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
|
||||
var_Dump($sSQL);
|
||||
$oDB->query($sSQL);
|
||||
echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
||||
}
|
||||
|
||||
$iFileSize = filesize($sImportFile);
|
||||
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
||||
|
||||
// Import the file
|
||||
$fCMDStartTime = time();
|
||||
echo $sCMDImport."\n";
|
||||
exec($sCMDImport, $sJunk, $iErrorLevel);
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
echo "Error: $iErrorLevel\n";
|
||||
exit($iErrorLevel);
|
||||
}
|
||||
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
|
||||
var_Dump($sSQL);
|
||||
$oDB->query($sSQL);
|
||||
echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
||||
$iFileSize = filesize($sImportFile);
|
||||
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
||||
|
||||
// Archive for debug?
|
||||
unlink($sImportFile);
|
||||
// Import the file
|
||||
$fCMDStartTime = time();
|
||||
echo $sCMDImport."\n";
|
||||
exec($sCMDImport, $sJunk, $iErrorLevel);
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
echo "Error: $iErrorLevel\n";
|
||||
exit($iErrorLevel);
|
||||
}
|
||||
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
|
||||
var_Dump($sSQL);
|
||||
$oDB->query($sSQL);
|
||||
echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
||||
|
||||
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
||||
// Archive for debug?
|
||||
unlink($sImportFile);
|
||||
|
||||
// Index file
|
||||
$sThisIndexCmd = $sCMDIndex;
|
||||
$fCMDStartTime = time();
|
||||
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
||||
|
||||
if (!$aResult['no-index'])
|
||||
{
|
||||
echo "$sThisIndexCmd\n";
|
||||
exec($sThisIndexCmd, $sJunk, $iErrorLevel);
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
echo "Error: $iErrorLevel\n";
|
||||
exit($iErrorLevel);
|
||||
}
|
||||
}
|
||||
// Index file
|
||||
$sThisIndexCmd = $sCMDIndex;
|
||||
$fCMDStartTime = time();
|
||||
|
||||
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
|
||||
var_Dump($sSQL);
|
||||
$oDB->query($sSQL);
|
||||
echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
||||
if (!$aResult['no-index'])
|
||||
{
|
||||
echo "$sThisIndexCmd\n";
|
||||
exec($sThisIndexCmd, $sJunk, $iErrorLevel);
|
||||
if ($iErrorLevel)
|
||||
{
|
||||
echo "Error: $iErrorLevel\n";
|
||||
exit($iErrorLevel);
|
||||
}
|
||||
}
|
||||
|
||||
$sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
|
||||
$oDB->query($sSQL);
|
||||
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
|
||||
var_Dump($sSQL);
|
||||
$oDB->query($sSQL);
|
||||
echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
||||
|
||||
$fDuration = time() - $fStartTime;
|
||||
echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
|
||||
if (!$aResult['import-osmosis-all']) exit(0);
|
||||
$sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
|
||||
$oDB->query($sSQL);
|
||||
|
||||
if ( CONST_Replication_Update_Interval > 60 )
|
||||
{
|
||||
$iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
|
||||
}
|
||||
else
|
||||
{
|
||||
$iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
|
||||
}
|
||||
echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
|
||||
sleep($iSleep);
|
||||
}
|
||||
}
|
||||
$fDuration = time() - $fStartTime;
|
||||
echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
|
||||
if (!$aResult['import-osmosis-all']) exit(0);
|
||||
|
||||
function getosmosistimestamp($sOsmosisConfigDirectory)
|
||||
{
|
||||
$sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
|
||||
preg_match('#timestamp=(.+)#', $sStateFile, $aResult);
|
||||
return str_replace('\:',':',$aResult[1]);
|
||||
}
|
||||
if ( CONST_Replication_Update_Interval > 60 )
|
||||
{
|
||||
$iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
|
||||
}
|
||||
else
|
||||
{
|
||||
$iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
|
||||
}
|
||||
echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
|
||||
sleep($iSleep);
|
||||
}
|
||||
}
|
||||
|
||||
function getosmosistimestamp($sOsmosisConfigDirectory)
|
||||
{
|
||||
$sStateFile = file_get_contents($sOsmosisConfigDirectory.'/state.txt');
|
||||
preg_match('#timestamp=(.+)#', $sStateFile, $aResult);
|
||||
return str_replace('\:',':',$aResult[1]);
|
||||
}
|
||||
|
||||
110
utils/warm.php
110
utils/warm.php
@@ -1,69 +1,69 @@
|
||||
#!/usr/bin/php -Cq
|
||||
<?php
|
||||
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
require_once(dirname(dirname(__FILE__)).'/settings/settings.php');
|
||||
require_once(CONST_BasePath.'/lib/init-cmd.php');
|
||||
ini_set('memory_limit', '800M');
|
||||
|
||||
$aCMDOptions = array(
|
||||
"Tools to warm nominatim db",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
array('reverse-only', '', 0, 1, 0, 0, 'bool', 'Warm reverse only'),
|
||||
array('search-only', '', 0, 1, 0, 0, 'bool', 'Warm reverse only'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
|
||||
$aCMDOptions = array(
|
||||
"Tools to warm nominatim db",
|
||||
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
|
||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||
array('reverse-only', '', 0, 1, 0, 0, 'bool', 'Warm reverse only'),
|
||||
array('search-only', '', 0, 1, 0, 0, 'bool', 'Warm reverse only'),
|
||||
);
|
||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
|
||||
|
||||
require_once(CONST_BasePath.'/lib/log.php');
|
||||
require_once(CONST_BasePath.'/lib/Geocode.php');
|
||||
require_once(CONST_BasePath.'/lib/PlaceLookup.php');
|
||||
require_once(CONST_BasePath.'/lib/ReverseGeocode.php');
|
||||
require_once(CONST_BasePath.'/lib/log.php');
|
||||
require_once(CONST_BasePath.'/lib/Geocode.php');
|
||||
require_once(CONST_BasePath.'/lib/PlaceLookup.php');
|
||||
require_once(CONST_BasePath.'/lib/ReverseGeocode.php');
|
||||
|
||||
$oDB =& getDB();
|
||||
$oDB =& getDB();
|
||||
|
||||
$bVerbose = $aResult['verbose'];
|
||||
$bVerbose = $aResult['verbose'];
|
||||
|
||||
if (!$aResult['search-only']) {
|
||||
if (!$aResult['search-only']) {
|
||||
|
||||
$oReverseGeocode = new ReverseGeocode($oDB);
|
||||
$oReverseGeocode->setZoom(20);
|
||||
$oPlaceLookup = new PlaceLookup($oDB);
|
||||
$oPlaceLookup->setIncludeAddressDetails(true);
|
||||
$oPlaceLookup->setLanguagePreference(array('en'));
|
||||
$oReverseGeocode = new ReverseGeocode($oDB);
|
||||
$oReverseGeocode->setZoom(20);
|
||||
$oPlaceLookup = new PlaceLookup($oDB);
|
||||
$oPlaceLookup->setIncludeAddressDetails(true);
|
||||
$oPlaceLookup->setLanguagePreference(array('en'));
|
||||
|
||||
echo "Warm reverse: ";
|
||||
if ($bVerbose) echo "\n";
|
||||
for($i = 0; $i < 1000; $i++) {
|
||||
$fLat = rand(-9000, 9000) / 100;
|
||||
$fLon = rand(-18000, 18000) / 100;
|
||||
if ($bVerbose) echo "$fLat, $fLon = ";
|
||||
$aLookup = $oReverseGeocode->lookup($fLat, $fLon);
|
||||
if ($aLookup && $aLookup['place_id'])
|
||||
{
|
||||
$aDetails = $oPlaceLookup->lookup((int)$aLookup['place_id'],
|
||||
$aLookup['type'], $aLookup['fraction']);
|
||||
if ($bVerbose) echo $aDetails['langaddress']."\n";
|
||||
}
|
||||
else echo ".";
|
||||
}
|
||||
echo "\n";
|
||||
}
|
||||
echo "Warm reverse: ";
|
||||
if ($bVerbose) echo "\n";
|
||||
for($i = 0; $i < 1000; $i++) {
|
||||
$fLat = rand(-9000, 9000) / 100;
|
||||
$fLon = rand(-18000, 18000) / 100;
|
||||
if ($bVerbose) echo "$fLat, $fLon = ";
|
||||
$aLookup = $oReverseGeocode->lookup($fLat, $fLon);
|
||||
if ($aLookup && $aLookup['place_id'])
|
||||
{
|
||||
$aDetails = $oPlaceLookup->lookup((int)$aLookup['place_id'],
|
||||
$aLookup['type'], $aLookup['fraction']);
|
||||
if ($bVerbose) echo $aDetails['langaddress']."\n";
|
||||
}
|
||||
else echo ".";
|
||||
}
|
||||
echo "\n";
|
||||
}
|
||||
|
||||
if (!$aResult['reverse-only']) {
|
||||
if (!$aResult['reverse-only']) {
|
||||
|
||||
$oGeocode =& new Geocode($oDB);
|
||||
$oGeocode =& new Geocode($oDB);
|
||||
|
||||
echo "Warm search: ";
|
||||
if ($bVerbose) echo "\n";
|
||||
$sSQL = 'select word from word where word is not null order by search_name_count desc limit 1000';
|
||||
foreach($oDB->getCol($sSQL) as $sWord) {
|
||||
if ($bVerbose) echo "$sWord = ";
|
||||
$oGeocode->setLanguagePreference(array('en'));
|
||||
$oGeocode->setQuery($sWord);
|
||||
$aSearchResults = $oGeocode->lookup();
|
||||
if ($bVerbose) echo $aSearchResults[0]['langaddress']."\n";
|
||||
else echo ".";
|
||||
}
|
||||
}
|
||||
echo "Warm search: ";
|
||||
if ($bVerbose) echo "\n";
|
||||
$sSQL = 'select word from word where word is not null order by search_name_count desc limit 1000';
|
||||
foreach($oDB->getCol($sSQL) as $sWord) {
|
||||
if ($bVerbose) echo "$sWord = ";
|
||||
$oGeocode->setLanguagePreference(array('en'));
|
||||
$oGeocode->setQuery($sWord);
|
||||
$aSearchResults = $oGeocode->lookup();
|
||||
if ($bVerbose) echo $aSearchResults[0]['langaddress']."\n";
|
||||
else echo ".";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user