hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-11 13:24:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve update interval computation

Browse Source 
Don't do anything if a downloaded diff is empty after all
(may be happening when an empty diff was published upstream).

Correctly compute the waiting interval before checking for new
data. As the interval is now computed based on the date of the
newest object in the database, the configured intervals need
to be adjusted slightly to take into account the time it takes
for the server to publish a diff.
This commit is contained in:
Sarah Hoffmann
2017-06-11 09:05:24 +02:00
parent 4e6c75040e
commit 896421fe9f
3 changed files with 33 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
settings/defaults.php
View File

@@ -69,11 +69,18 @@ if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true);
@define('CONST_Tablespace_Aux_Data', false); @define('CONST_Tablespace_Aux_Data', false);
@define('CONST_Tablespace_Aux_Index', false); @define('CONST_Tablespace_Aux_Index', false);
// Replication settings //// Replication settings
// Base URL of replication service
@define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute'); @define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute');
@define('CONST_Replication_Max_Diff_size', '30'); // MB of update data to download per batch
@define('CONST_Replication_Update_Interval', '60'); // How often upstream publishes diffs // Maximum size in MB of data to download per batch
@define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet @define('CONST_Replication_Max_Diff_size', '30');
// How long until the service publishes the next diff
// (relative to the age of data in the diff).
@define('CONST_Replication_Update_Interval', '75');
// How long to sleep when no update could be found
@define('CONST_Replication_Recheck_Interval', '60');
// Website settings // Website settings
@define('CONST_NoAccessControl', true); @define('CONST_NoAccessControl', true);

5
utils/osm_file_date.py
View File

@@ -28,6 +28,7 @@ if __name__ == '__main__':
h.apply_file(sys.argv[1]) h.apply_file(sys.argv[1])
if h.filedate is None:
exit(5)
print(h.filedate) print(h.filedate)

31
utils/update.php
View File

@@ -241,9 +241,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
while (true) { while (true) {
$fStartTime = time(); $fStartTime = time();
$iFileSize = 1001; $aLastState = chksql($oDB->getRow('SELECT *, EXTRACT (EPOCH FROM lastimportdate) as unix_ts FROM import_status'));
$aLastState = chksql($oDB->getRow('SELECT * FROM import_status'));
if (!$aLastState['sequence_id']) { if (!$aLastState['sequence_id']) {
echo "Updates not set up. Please run ./utils/update.php --init-updates.\n"; echo "Updates not set up. Please run ./utils/update.php --init-updates.\n";
@@ -257,7 +255,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
if ($aLastState['indexed'] == 't') { if ($aLastState['indexed'] == 't') {
// Sleep if the update interval has not yet been reached. // Sleep if the update interval has not yet been reached.
$fNextUpdate = $aLastState['lastimportdate'] + CONST_Replication_Update_Interval; $fNextUpdate = $aLastState['unix_ts'] + CONST_Replication_Update_Interval;
if ($fNextUpdate > $fStartTime) { if ($fNextUpdate > $fStartTime) {
$iSleepTime = $fNextUpdate - $fStartTime; $iSleepTime = $fNextUpdate - $fStartTime;
echo "Waiting for next update for $iSleepTime sec."; echo "Waiting for next update for $iSleepTime sec.";
@@ -284,6 +282,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
} }
} while ($iResult); } while ($iResult);
// get the newest object from the diff file
$sBatchEnd = 0;
$iRet = 0;
exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
if ($iRet == 5) {
echo "Diff file is empty. skipping import.\n";
if (!$aResult['import-osmosis-all']) {
exit(0);
} else {
continue;
}
}
if ($iRet != 0) {
fail('Error getting date from diff file.');
}
$sBatchEnd = $sBatchEnd[0];
// Import the file // Import the file
$fCMDStartTime = time(); $fCMDStartTime = time();
echo $sCMDImport."\n"; echo $sCMDImport."\n";
@@ -296,14 +311,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
// write the update logs // write the update logs
$iFileSize = filesize($sImportFile); $iFileSize = filesize($sImportFile);
// get the newest object from the diff file
$sBatchEnd = 0;
$iRet = 0;
exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet);
if ($iRet != 0) {
fail('Error getting date from diff file.');
}
$sBatchEnd = $sBatchEnd[0];
$sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')"; $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
var_Dump($sSQL); var_Dump($sSQL);
chksql($oDB->query($sSQL)); chksql($oDB->query($sSQL));