hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-15 19:07:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

prune list of blocked items if too large, different costs for different page types

Browse Source
This commit is contained in:
Brian Quinion
2012-12-08 21:39:24 +00:00
parent 539d187b5c
commit 274f381052
7 changed files with 156 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
lib/init.php
View File

@@ -4,6 +4,7 @@
require_once(CONST_BasePath.'/settings/settings.php');
require_once(CONST_BasePath.'/lib/lib.php');
require_once(CONST_BasePath.'/lib/leakybucket.php');
require_once(CONST_BasePath.'/lib/db.php');
if (get_magic_quotes_gpc())

137
lib/leakybucket.php Normal file
View File

@@ -0,0 +1,137 @@
<?php
function getBucketMemcache()
{
static $m;
if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
if (!isset($m))
{
$m = new Memcached();
$m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
}
return $m;
}
function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
{
$m = getBucketMemcache();
if (!$m) return 0;
$iMaxVal = 0;
$t = time();
foreach($asKey as $sKey)
{
$aCurrentBlock = $m->get($sKey);
if (!$aCurrentBlock)
{
$aCurrentBlock = array($iRequestCost, $t);
}
else
{
// add RequestCost
// remove leak * the time since the last request
$aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
$aCurrentBlock[1] = $t;
}
if ($aCurrentBlock[0] <= 0)
{
$m->delete($sKey);
}
else
{
// If we have hit the threshold stop and record this to the block list
if ($aCurrentBlock[0] >= $iThreshold)
{
$aCurrentBlock[0] = $iThreshold;
// Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
$i = 10;
for($i = 0; $i < 10; $i++)
{
$aBlockedList = $m->get('blockedList', null, $hCasToken);
if (!$aBlockedList)
{
$aBlockedList = array();
$m->add('blockedList', $aBlockedList);
$aBlockedList = $m->get('blockedList', null, $hCasToken);
}
if (!isset($aBlockedList[$sKey]))
{
$aBlockedList[$sKey] = array(1, $t);
}
else
{
$aBlockedList[$sKey][0]++;
$aBlockedList[$sKey][1] = $t;
}
if (sizeof($aBlockedList) > CONST_ConnectionBucket_MaxBlockList)
{
uasort($aBlockedList, 'byValue1');
$aBlockedList = array_slice($aBlockedList, 0, CONST_ConnectionBucket_MaxBlockList);
}
$x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
if ($x) break;
}
}
// Only keep in memcache until the time it would have expired (to avoid clutering memcache)
$m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
}
// Bucket result in the largest bucket we find
$iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
}
return $iMaxVal;
}
function byValue1($a, $b)
{
if ($a[1] == $b[1])
{
return 0;
}
return ($a[1] > $b[1]) ? -1 : 1;
}
function byLastBlockTime($a, $b)
{
if ($a['lastBlockTimestamp'] == $b['lastBlockTimestamp'])
{
return 0;
}
return ($a['lastBlockTimestamp'] > $b['lastBlockTimestamp']) ? -1 : 1;
}
function getBucketBlocks()
{
$m = getBucketMemcache();
if (!$m) return null;
$t = time();
$aBlockedList = $m->get('blockedList', null, $hCasToken);
if (!$aBlockedList) $aBlockedList = array();
foreach($aBlockedList as $sKey => $aDetails)
{
$aCurrentBlock = $m->get($sKey);
if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
$iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
$aBlockedList[$sKey] = array(
'totalBlocks' => $aDetails[0],
'lastBlockTimestamp' => $aDetails[1],
'currentBucketSize' => $iCurrentBucketSize,
'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
);
}
uasort($aBlockedList, 'byLastBlockTime');
return $aBlockedList;
}
function clearBucketBlocks()
{
$m = getBucketMemcache();
if (!$m) return false;
$m->delete('blockedList');
return true;
}

1
settings/settings.php
View File

@@ -17,6 +17,7 @@
// Connection buckets to rate limit people being nasty
@define('CONST_ConnectionBucket_MemcacheServerAddress', false);
@define('CONST_ConnectionBucket_MemcacheServerPort', 11211);
@define('CONST_ConnectionBucket_MaxBlockList', 100);
@define('CONST_ConnectionBucket_LeakRate', 1);
@define('CONST_ConnectionBucket_BlockLimit', 10);
@define('CONST_ConnectionBucket_WaitLimit', 6);

14
utils/blocks.php
View File

@@ -11,6 +11,7 @@
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),
array('flush', '', 0, 1, 0, 0, 'bool', 'Flush all blocks / stats'),
);
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
@@ -28,11 +29,13 @@
$aBlocks = getBucketBlocks();
echo "\n";
printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Req Blocked", "Last Block Time");
printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-16s-|-%'-31s\n", "", "", "", "", "", "");
printf(" %-40s | %12s | %7s | %13s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Block Time");
printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s\n", "", "", "", "", "");
foreach($aBlocks as $sKey => $aDetails)
{
printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", $sKey, $aDetails['totalBlocks'], (int)$aDetails['currentBucketSize'], $aDetails['lastRequestBlocked']?'Y':'N', $aDetails['currentlyBlocked']?'Y':'N', date("r", $aDetails['lastBlockTimestamp']));
printf(" %-40s | %12s | %7s | %13s | %31s\n", $sKey, $aDetails['totalBlocks'],
(int)$aDetails['currentBucketSize'], $aDetails['currentlyBlocked']?'Y':'N',
date("r", $aDetails['lastBlockTimestamp']));
}
echo "\n";
}
@@ -42,3 +45,8 @@
$m->set('sleepCounter', 0);
clearBucketBlocks();
}
if ($aResult['flush'])
{
$m->flush();
}

2
website/details.php
View File

@@ -1,4 +1,6 @@
<?php
@define('CONST_ConnectionBucket_PageType', 'Details');
require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
require_once(CONST_BasePath.'/lib/log.php');

2
website/reverse.php
View File

@@ -1,4 +1,6 @@
<?php
@define('CONST_ConnectionBucket_PageType', 'Reverse');
require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
require_once(CONST_BasePath.'/lib/log.php');

2
website/search.php
View File

@@ -1,4 +1,6 @@
<?php
@define('CONST_ConnectionBucket_PageType', 'Search');
require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
require_once(CONST_BasePath.'/lib/log.php');