safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

fix typo for baseUrl when configuring configuration.txt

Browse Source 
use osmosis --read-replication-lag to determine if there are changes before trying to process updates, useful when we are tracking hourly or daily replication updates

set CONST_Replication_Recheck_Interval to 60

skip lag check if CONST_Replication_Update_Interval > 60, for minutelies there's always new diffs to process

use tabs for indent

change sleep for non-minutely updates so that we dont drift tto much or poll excessively

unset $aReplicationLag before each exec attempt

unset $aReplicationLag inside while loop
This commit is contained in:
IrlJidel
2013-03-26 14:49:35 +00:00
parent f2b2a71485
commit 3eea62363d
3 changed files with 40 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
settings/settings.php
View File

@@ -16,8 +16,12 @@
@define('CONST_Path_Postgresql_Postgis', CONST_Path_Postgresql_Contrib.'/postgis-'.CONST_Postgis_Version);
@define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql');
@define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
// Replication settings
@define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute');
@define('CONST_Replication_MaxInterval', '3600');
@define('CONST_Replication_Update_Interval', '60'); // How often upstream publishes diffs
@define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet
// Connection buckets to rate limit people being nasty
@define('CONST_ConnectionBucket_MemcacheServerAddress', false);

2
utils/setup.php
View File

@@ -505,7 +505,7 @@
passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
// update osmosis configuration.txt with our settings
passthru("sed -i 's!baseUrl=.*!baseUrL=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
}

38
utils/update.php
View File

@@ -353,6 +353,7 @@
$sOsmosisCMD = CONST_Osmosis_Binary;
$sOsmosisConfigDirectory = CONST_BasePath.'/settings';
$sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
$sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
$sCMDImport = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$aDSNInfo['database'].' '.$sImportFile;
$sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'];
if (!$aResult['no-npi']) {
@@ -372,7 +373,30 @@
// {
if (!file_exists($sImportFile))
{
// Use osmosis to download the file
// First check if there are new updates published (except for minutelies - there's always new diffs to process)
if ( CONST_Replication_Update_Interval > 60 )
{
unset($aReplicationLag);
exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
while ($iErrorLevel == 1 || $aReplicationLag[0] < 1)
{
if ($iErrorLevel)
{
echo "Error: $iErrorLevel. ";
echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
}
else
{
echo ".";
}
sleep(CONST_Replication_Recheck_Interval);
unset($aReplicationLag);
exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
}
// There are new replication files - use osmosis to download the file
echo "\nReplication Delay is ".$aReplicationLag[0]."\n";
}
$fCMDStartTime = time();
echo $sCMDDownload."\n";
exec($sCMDDownload, $sJunk, $iErrorLevel);
@@ -483,8 +507,16 @@
echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n";
if (!$aResult['import-osmosis-all']) exit;
echo "Sleeping ".max(0,60-$fDuration)." seconds\n";
sleep(max(0,60-$fDuration));
if ( CONST_Replication_Update_Interval > 60 )
{
$iSleep = round(CONST_Replication_Update_Interval*0.8);
}
else
{
$iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
}
echo "Sleeping $iSleep seconds\n";
sleep($iSleep);
}
}