prune list of blocked items if too large, different costs for different page types

lib/init.php

@@ -4,6 +4,7 @@
 
 	require_once(CONST_BasePath.'/settings/settings.php');
 	require_once(CONST_BasePath.'/lib/lib.php');
+	require_once(CONST_BasePath.'/lib/leakybucket.php');
 	require_once(CONST_BasePath.'/lib/db.php');
 
 	if (get_magic_quotes_gpc())

lib/leakybucket.php

@@ -0,0 +1,137 @@
+<?php
+
+	function getBucketMemcache()
+	{
+		static $m;
+
+		if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
+		if (!isset($m))
+		{
+		        $m = new Memcached();
+        		$m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
+		}
+		return $m;
+	}
+
+	function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return 0;
+
+		$iMaxVal = 0;
+	        $t = time();
+
+		foreach($asKey as $sKey)
+		{
+		        $aCurrentBlock = $m->get($sKey);
+	        	if (!$aCurrentBlock)
+			{
+		                $aCurrentBlock = array($iRequestCost, $t);
+        		}
+			else
+			{
+				// add RequestCost
+				// remove leak * the time since the last request 
+	        		$aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
+		        	$aCurrentBlock[1] = $t;
+			}
+
+	        	if ($aCurrentBlock[0] <= 0)
+			{
+                		$m->delete($sKey);
+		        }
+			else
+			{
+				// If we have hit the threshold stop and record this to the block list
+				if ($aCurrentBlock[0] >= $iThreshold)
+				{
+					$aCurrentBlock[0] = $iThreshold;
+
+					// Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
+					$i = 10;
+					for($i = 0; $i < 10; $i++)
+					{
+						$aBlockedList = $m->get('blockedList', null, $hCasToken);
+						if (!$aBlockedList)
+						{
+							$aBlockedList = array();
+							$m->add('blockedList', $aBlockedList);
+							$aBlockedList = $m->get('blockedList', null, $hCasToken);
+						}
+						if (!isset($aBlockedList[$sKey]))
+						{
+							$aBlockedList[$sKey] = array(1, $t);
+						}
+						else
+						{
+							$aBlockedList[$sKey][0]++;
+							$aBlockedList[$sKey][1] = $t;
+						}
+						if (sizeof($aBlockedList) > CONST_ConnectionBucket_MaxBlockList)
+						{
+							uasort($aBlockedList, 'byValue1');
+							$aBlockedList = array_slice($aBlockedList, 0, CONST_ConnectionBucket_MaxBlockList);
+						}
+						$x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
+						if ($x) break;
+					}
+				}
+				// Only keep in memcache until the time it would have expired (to avoid clutering memcache)
+	                	$m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
+			}
+
+			// Bucket result in the largest bucket we find
+			$iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
+		}
+
+		return $iMaxVal;
+        }
+
+	function byValue1($a, $b)
+	{
+		if ($a[1] == $b[1])
+		{
+			return 0;
+		}
+		return ($a[1] > $b[1]) ? -1 : 1;
+	}
+
+	function byLastBlockTime($a, $b)
+	{
+		if ($a['lastBlockTimestamp'] == $b['lastBlockTimestamp'])
+		{
+			return 0;
+		}
+		return ($a['lastBlockTimestamp'] > $b['lastBlockTimestamp']) ? -1 : 1;
+	}
+
+	function getBucketBlocks()
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return null;
+	        $t = time();
+		$aBlockedList = $m->get('blockedList', null, $hCasToken);
+		if (!$aBlockedList) $aBlockedList = array();
+		foreach($aBlockedList as $sKey => $aDetails)
+		{
+			$aCurrentBlock = $m->get($sKey);
+			if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
+			$iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
+			$aBlockedList[$sKey] = array(
+				'totalBlocks' => $aDetails[0],
+				'lastBlockTimestamp' => $aDetails[1],
+				'currentBucketSize' => $iCurrentBucketSize,
+				'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
+				);
+		}
+		uasort($aBlockedList, 'byLastBlockTime');
+		return $aBlockedList;
+	}
+
+	function clearBucketBlocks()
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return false;
+		$m->delete('blockedList');
+		return true;
+	}

settings/settings.php

@@ -17,6 +17,7 @@
 	// Connection buckets to rate limit people being nasty
 	@define('CONST_ConnectionBucket_MemcacheServerAddress', false);
 	@define('CONST_ConnectionBucket_MemcacheServerPort', 11211);
+	@define('CONST_ConnectionBucket_MaxBlockList', 100);
 	@define('CONST_ConnectionBucket_LeakRate', 1);
 	@define('CONST_ConnectionBucket_BlockLimit', 10);
 	@define('CONST_ConnectionBucket_WaitLimit', 6);

utils/blocks.php

@@ -11,6 +11,7 @@
 		array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
 		array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
 		array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),
+		array('flush', '', 0, 1, 0, 0, 'bool', 'Flush all blocks / stats'),
 	);
 	getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
 
@@ -28,11 +29,13 @@
 
 		$aBlocks = getBucketBlocks();
 		echo "\n";
-		printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Req Blocked", "Last Block Time");
+		printf(" %-40s | %12s | %7s | %13s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Block Time");
-		printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-16s-|-%'-31s\n", "", "", "", "", "", "");
+		printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s\n", "", "", "", "", "");
 		foreach($aBlocks as $sKey => $aDetails)
 		{
-			printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", $sKey, $aDetails['totalBlocks'], (int)$aDetails['currentBucketSize'], $aDetails['lastRequestBlocked']?'Y':'N', $aDetails['currentlyBlocked']?'Y':'N', date("r", $aDetails['lastBlockTimestamp']));
+			printf(" %-40s | %12s | %7s | %13s | %31s\n", $sKey, $aDetails['totalBlocks'], 
+				(int)$aDetails['currentBucketSize'], $aDetails['currentlyBlocked']?'Y':'N', 
+				date("r", $aDetails['lastBlockTimestamp']));
 		}
 		echo "\n";
 	}
@@ -42,3 +45,8 @@
 		$m->set('sleepCounter', 0);
 		clearBucketBlocks();
 	}
+
+	if ($aResult['flush'])
+	{
+		$m->flush();
+	}

website/details.php

@@ -1,4 +1,6 @@
 <?php
+	@define('CONST_ConnectionBucket_PageType', 'Details');
+
         require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
         require_once(CONST_BasePath.'/lib/log.php');
 

website/reverse.php

@@ -1,4 +1,6 @@
 <?php
+	@define('CONST_ConnectionBucket_PageType', 'Reverse');
+
 	require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
 	require_once(CONST_BasePath.'/lib/log.php');
 

website/search.php

@@ -1,4 +1,6 @@
 <?php
+	@define('CONST_ConnectionBucket_PageType', 'Search');
+
 	require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
 	require_once(CONST_BasePath.'/lib/log.php');