hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-26 11:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove IP blocking and rate-limiting code

Browse Source 
Closes #472.
This commit is contained in:
Sarah Hoffmann
2016-07-18 14:41:09 +02:00
parent 975c30afba
commit 4f75f4cb6e
8 changed files with 1 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
lib/init-website.php
View File

@@ -13,47 +13,5 @@
} }
if ($_SERVER['REQUEST_METHOD'] == 'OPTIONS') exit; if ($_SERVER['REQUEST_METHOD'] == 'OPTIONS') exit;
$aBucketKeys = array();
if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST)));
if (isset($_SERVER["REMOTE_ADDR"])) $aBucketKeys[] = $_SERVER["REMOTE_ADDR"];
if (isset($_GET["email"])) $aBucketKeys[] = $_GET["email"];
$fBucketVal = doBucket($aBucketKeys,
(defined('CONST_ConnectionBucket_PageType')?constant('CONST_ConnectionBucket_Cost_'.CONST_ConnectionBucket_PageType):1) + user_busy_cost(),
CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
if ($fBucketVal > CONST_ConnectionBucket_WaitLimit && $fBucketVal < CONST_ConnectionBucket_BlockLimit)
{
$m = getBucketMemcache();
$iCurrentSleeping = $m->increment('sleepCounter');
if (false === $iCurrentSleeping)
{
$m->add('sleepCounter', 0);
$iCurrentSleeping = $m->increment('sleepCounter');
}
if ($iCurrentSleeping >= CONST_ConnectionBucket_MaxSleeping || isBucketSleeping($aBucketKeys))
{
// Too many threads sleeping already. This becomes a hard block.
$fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_BlockLimit, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
}
else
{
setBucketSleeping($aBucketKeys, true);
sleep(($fBucketVal - CONST_ConnectionBucket_WaitLimit)/CONST_ConnectionBucket_LeakRate);
$fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
setBucketSleeping($aBucketKeys, false);
}
$m->decrement('sleepCounter');
}
if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit)
{
header("HTTP/1.0 429 Too Many Requests");
echo "Your IP has been blocked. \n";
echo CONST_BlockMessage;
exit;
}
header('Content-type: text/html; charset=utf-8'); header('Content-type: text/html; charset=utf-8');

1
lib/init.php
View File

@@ -1,7 +1,6 @@
<?php <?php
require_once(CONST_BasePath.'/lib/lib.php'); require_once(CONST_BasePath.'/lib/lib.php');
require_once(CONST_BasePath.'/lib/leakybucket.php');
require_once(CONST_BasePath.'/lib/db.php'); require_once(CONST_BasePath.'/lib/db.php');
if (get_magic_quotes_gpc()) if (get_magic_quotes_gpc())

168
lib/leakybucket.php
View File

@@ -1,168 +0,0 @@
<?php
function getBucketMemcache()
{
static $m;
if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
if (!isset($m))
{
$m = new Memcached();
$m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
}
return $m;
}
function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
{
$m = getBucketMemcache();
if (!$m) return 0;
$iMaxVal = 0;
$t = time();
foreach($asKey as $sKey)
{
$aCurrentBlock = $m->get($sKey);
if (!$aCurrentBlock)
{
$aCurrentBlock = array($iRequestCost, $t, false);
}
else
{
// add RequestCost
// remove leak * the time since the last request
$aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
$aCurrentBlock[1] = $t;
}
if ($aCurrentBlock[0] <= 0)
{
$m->delete($sKey);
}
else
{
// If we have hit the threshold stop and record this to the block list
if ($aCurrentBlock[0] >= $iThreshold)
{
$aCurrentBlock[0] = $iThreshold;
// Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
$i = 10;
for($i = 0; $i < 10; $i++)
{
$aBlockedList = $m->get('blockedList', null, $hCasToken);
if (!$aBlockedList)
{
$aBlockedList = array();
$m->add('blockedList', $aBlockedList);
$aBlockedList = $m->get('blockedList', null, $hCasToken);
}
if (!isset($aBlockedList[$sKey]))
{
$aBlockedList[$sKey] = array(1, $t);
}
else
{
$aBlockedList[$sKey][0]++;
$aBlockedList[$sKey][1] = $t;
}
if (sizeof($aBlockedList) > CONST_ConnectionBucket_MaxBlockList)
{
uasort($aBlockedList, 'byValue1');
$aBlockedList = array_slice($aBlockedList, 0, CONST_ConnectionBucket_MaxBlockList);
}
$x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
if ($x) break;
}
}
// Only keep in memcache until the time it would have expired (to avoid clutering memcache)
$m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
}
// Bucket result in the largest bucket we find
$iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
}
return $iMaxVal;
}
function isBucketSleeping($asKey)
{
$m = getBucketMemcache();
if (!$m) return false;
foreach($asKey as $sKey)
{
$aCurrentBlock = $m->get($sKey);
if ($aCurrentBlock[2]) return true;
}
return false;
}
function setBucketSleeping($asKey, $bVal)
{
$m = getBucketMemcache();
if (!$m) return false;
$iMaxVal = 0;
$t = time();
foreach($asKey as $sKey)
{
$aCurrentBlock = $m->get($sKey);
$aCurrentBlock[2] = $bVal;
$m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/CONST_ConnectionBucket_LeakRate);
}
return true;
}
function byValue1($a, $b)
{
if ($a[1] == $b[1])
{
return 0;
}
return ($a[1] > $b[1]) ? -1 : 1;
}
function byLastBlockTime($a, $b)
{
if ($a['lastBlockTimestamp'] == $b['lastBlockTimestamp'])
{
return 0;
}
return ($a['lastBlockTimestamp'] > $b['lastBlockTimestamp']) ? -1 : 1;
}
function getBucketBlocks()
{
$m = getBucketMemcache();
if (!$m) return null;
$t = time();
$aBlockedList = $m->get('blockedList', null, $hCasToken);
if (!$aBlockedList) $aBlockedList = array();
foreach($aBlockedList as $sKey => $aDetails)
{
$aCurrentBlock = $m->get($sKey);
if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
$iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
$aBlockedList[$sKey] = array(
'totalBlocks' => $aDetails[0],
'lastBlockTimestamp' => $aDetails[1],
'isSleeping' => (isset($aCurrentBlock[2])?$aCurrentBlock[2]:false),
'currentBucketSize' => $iCurrentBucketSize,
'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
);
}
uasort($aBlockedList, 'byLastBlockTime');
return $aBlockedList;
}
function clearBucketBlocks()
{
$m = getBucketMemcache();
if (!$m) return false;
$m->delete('blockedList');
return true;
}

8
lib/lib.php
View File

@@ -9,14 +9,6 @@
} }
function getLoadAverage()
{
$sLoadAverage = file_get_contents('/proc/loadavg');
$aLoadAverage = explode(' ',$sLoadAverage);
return (float)$aLoadAverage[0];
}
function getProcessorCount() function getProcessorCount()
{ {
$sCPU = file_get_contents('/proc/cpuinfo'); $sCPU = file_get_contents('/proc/cpuinfo');

26
settings/settings.php
View File

@@ -62,34 +62,8 @@
@define('CONST_Replication_Update_Interval', '60'); // How often upstream publishes diffs @define('CONST_Replication_Update_Interval', '60'); // How often upstream publishes diffs
@define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet @define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet
// Connection buckets to rate limit people being nasty
@define('CONST_ConnectionBucket_MemcacheServerAddress', false);
@define('CONST_ConnectionBucket_MemcacheServerPort', 11211);
@define('CONST_ConnectionBucket_MaxBlockList', 100);
@define('CONST_ConnectionBucket_LeakRate', 1);
@define('CONST_ConnectionBucket_BlockLimit', 10);
@define('CONST_ConnectionBucket_WaitLimit', 6);
@define('CONST_ConnectionBucket_MaxSleeping', 10);
@define('CONST_ConnectionBucket_Cost_Reverse', 1);
@define('CONST_ConnectionBucket_Cost_Search', 2);
@define('CONST_ConnectionBucket_Cost_Details', 3);
@define('CONST_ConnectionBucket_Cost_Status', 1);
// Override this function to add an adjustment factor to the cost
// based on server load. e.g. getBlockingProcesses
if (!function_exists('user_busy_cost'))
{
function user_busy_cost()
{
return 0;
}
}
// Website settings // Website settings
@define('CONST_NoAccessControl', true); @define('CONST_NoAccessControl', true);
@define('CONST_BlockedIPs', '');
@define('CONST_BulkUserIPs', '');
@define('CONST_BlockMessage', ''); // additional info to show for blocked IPs
@define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/'); @define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/');
// Language to assume when none is supplied with the query. // Language to assume when none is supplied with the query.

9
website/details.php
View File

@@ -7,14 +7,7 @@
require_once(CONST_BasePath.'/lib/output.php'); require_once(CONST_BasePath.'/lib/output.php');
$sOutputFormat = 'html'; $sOutputFormat = 'html';
/*
$fLoadAvg = getLoadAverage();
if ($fLoadAvg > 3)
{
echo "Page temporarily blocked due to high server load\n";
exit;
}
*/
ini_set('memory_limit', '200M'); ini_set('memory_limit', '200M');
$oDB =& getDB(); $oDB =& getDB();

12
website/lookup.php
View File

@@ -7,18 +7,6 @@
require_once(CONST_BasePath.'/lib/PlaceLookup.php'); require_once(CONST_BasePath.'/lib/PlaceLookup.php');
require_once(CONST_BasePath.'/lib/output.php'); require_once(CONST_BasePath.'/lib/output.php');
if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
{
$fLoadAvg = getLoadAverage();
if ($fLoadAvg > 2) sleep(60);
if ($fLoadAvg > 4) sleep(120);
if ($fLoadAvg > 6)
{
userError("Bulk User: Temporary block due to high server load");
exit;
}
}
$oDB =& getDB(); $oDB =& getDB();
ini_set('memory_limit', '200M'); ini_set('memory_limit', '200M');

13
website/reverse.php
View File

@@ -8,19 +8,6 @@
require_once(CONST_BasePath.'/lib/ReverseGeocode.php'); require_once(CONST_BasePath.'/lib/ReverseGeocode.php');
require_once(CONST_BasePath.'/lib/output.php'); require_once(CONST_BasePath.'/lib/output.php');
if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
{
$fLoadAvg = getLoadAverage();
if ($fLoadAvg > 2) sleep(60);
if ($fLoadAvg > 4) sleep(120);
if ($fLoadAvg > 6)
{
echo "Bulk User: Temporary block due to high server load\n";
exit;
}
}
$bAsGeoJSON = getParamBool('polygon_geojson'); $bAsGeoJSON = getParamBool('polygon_geojson');
$bAsKML = getParamBool('polygon_kml'); $bAsKML = getParamBool('polygon_kml');
$bAsSVG = getParamBool('polygon_svg'); $bAsSVG = getParamBool('polygon_svg');