hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-16 15:47:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

rate limiting using memcache

Browse Source
This commit is contained in:
Brian Quinion
2012-12-08 17:38:10 +00:00
parent bde251f2a2
commit e03ae6d7eb
4 changed files with 188 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
lib/init-website.php
View File

@@ -1,5 +1,8 @@
<?php
//echo "<a href=\"http://localhost/nominatim/search.php?format=xml&addressdetails=1&accept-language=en&street=&suburb=&city=Dammam&county=&state=&country=SA&postcode=&\">x</a>";
//exit;
//phpinfo();
//exit;
require_once('init.php');
if (CONST_ClosedForIndexing && strpos(CONST_ClosedForIndexingExceptionIPs, ','.$_SERVER["REMOTE_ADDR"].',') === false)
@@ -8,7 +11,16 @@
exit;
}
if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
$aBucketKeys = array();
if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST)));
if (isset($_SERVER["REMOTE_ADDR"])) $aBucketKeys[] = $_SERVER["REMOTE_ADDR"];
if (isset($_GET["email"])) $aBucketKeys[] = $_GET["email"];
$fBucketVal = doBucket($aBucketKeys,
(defined('CONST_ConnectionBucket_PageType')?constant('CONST_ConnectionBucket_Cost_'.CONST_ConnectionBucket_PageType):1) + user_busy_cost(),
CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit)
{
echo "Your IP has been blocked. \n";
echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n";
@@ -16,4 +28,10 @@
exit;
}
if ($fBucketVal > CONST_ConnectionBucket_WaitLimit)
{
sleep(($fBucketVal - CONST_ConnectionBucket_WaitLimit)/CONST_ConnectionBucket_LeakRate);
}
var_dump($fBucketVal);
exit;
header('Content-type: text/html; charset=utf-8');

109
lib/lib.php
View File

@@ -60,7 +60,7 @@
{
$sLoadAverage = file_get_contents('/proc/loadavg');
$aLoadAverage = explode(' ',$sLoadAverage);
return (int)$aLoadAverage[0];
return (float)$aLoadAverage[0];
}
function getProcessorCount()
@@ -917,3 +917,110 @@
return true;
}
function getBucketMemcache()
{
if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
$m = new Memcached();
$m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
return $m;
}
function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
{
$m = getBucketMemcache();
if (!$m) return 0;
$iMaxVal = 0;
$t = time();
foreach($asKey as $sKey)
{
$aCurrentBlock = $m->get($sKey);
if (!$aCurrentBlock)
{
$aCurrentBlock = array($iRequestCost, $t);
}
else
{
// add RequestCost
// remove leak * the time since the last request
$aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
$aCurrentBlock[1] = $t;
}
if ($aCurrentBlock[0] <= 0)
{
$m->delete($sKey);
}
else
{
// If we have hit the threshold stop and record this to the block list
if ($aCurrentBlock[0] >= $iThreshold)
{
$aCurrentBlock[0] = $iThreshold;
// Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
$i = 10;
for($i = 0; $i < 10; $i++)
{
$aBlockedList = $m->get('blockedList', null, $hCasToken);
if (!$aBlockedList)
{
$aBlockedList = array();
$m->add('blockedList', $aBlockedList);
$aBlockedList = $m->get('blockedList', null, $hCasToken);
}
if (!isset($aBlockedList[$sKey]))
{
$aBlockedList[$sKey] = array(1, $t);
}
else
{
$aBlockedList[$sKey][0]++;
$aBlockedList[$sKey][1] = $t;
}
$x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
if ($x) break;
}
}
// Only keep in memcache until the time it would have expired (to avoid clutering memcache)
$m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
}
// Bucket result in the largest bucket we find
$iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
}
return $iMaxVal;
}
function getBucketBlocks()
{
$m = getBucketMemcache();
if (!$m) return null;
$t = time();
$aBlockedList = $m->get('blockedList', null, $hCasToken);
if (!$aBlockedList) $aBlockedList = array();
foreach($aBlockedList as $sKey => $aDetails)
{
$aCurrentBlock = $m->get($sKey);
if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
$iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
$aBlockedList[$sKey] = array(
'totalBlocks' => $aDetails[0],
'lastBlockTimestamp' => $aDetails[1],
'currentBucketSize' => $iCurrentBucketSize,
'lastRequestBlocked' => $aCurrentBlock[0] >= CONST_ConnectionBucket_BlockLimit,
'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
);
}
return $aBlockedList;
}
function clearBucketBlocks()
{
$m = getBucketMemcache();
if (!$m) return false;
$m->delete('blockedList');
return true;
}