rate limiting using memcache

lib/init-website.php

@@ -1,5 +1,8 @@
 <?php
-
+//echo "<a href=\"http://localhost/nominatim/search.php?format=xml&addressdetails=1&accept-language=en&street=&suburb=&city=Dammam&county=&state=&country=SA&postcode=&\">x</a>";
+//exit;
+//phpinfo();
+//exit;
 	require_once('init.php');
 
 	if (CONST_ClosedForIndexing && strpos(CONST_ClosedForIndexingExceptionIPs, ','.$_SERVER["REMOTE_ADDR"].',') === false)
@@ -8,7 +11,16 @@
 		exit;
 	}
 
-	if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
+	$aBucketKeys = array();
+	if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST)));
+	if (isset($_SERVER["REMOTE_ADDR"])) $aBucketKeys[] = $_SERVER["REMOTE_ADDR"];
+	if (isset($_GET["email"])) $aBucketKeys[] = $_GET["email"];
+
+	$fBucketVal = doBucket($aBucketKeys, 
+			(defined('CONST_ConnectionBucket_PageType')?constant('CONST_ConnectionBucket_Cost_'.CONST_ConnectionBucket_PageType):1) + user_busy_cost(),
+			CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
+
+	if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit)
 	{
 		echo "Your IP has been blocked. \n";
 		echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n";
@@ -16,4 +28,10 @@
 		exit;
 	}
 
+	if ($fBucketVal > CONST_ConnectionBucket_WaitLimit)
+	{
+		sleep(($fBucketVal - CONST_ConnectionBucket_WaitLimit)/CONST_ConnectionBucket_LeakRate);
+	}
+var_dump($fBucketVal);
+exit;
 	header('Content-type: text/html; charset=utf-8');

lib/lib.php

@@ -60,7 +60,7 @@
 	{
 		$sLoadAverage = file_get_contents('/proc/loadavg');
                 $aLoadAverage = explode(' ',$sLoadAverage);
-		return (int)$aLoadAverage[0];
+		return (float)$aLoadAverage[0];
 	}
 
 	function getProcessorCount()
@@ -917,3 +917,110 @@
                 return true;
         }
 
+	function getBucketMemcache()
+	{
+		if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
+	        $m = new Memcached();
+        	$m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
+		return $m;
+	}
+
+	function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return 0;
+
+		$iMaxVal = 0;
+	        $t = time();
+
+		foreach($asKey as $sKey)
+		{
+		        $aCurrentBlock = $m->get($sKey);
+	        	if (!$aCurrentBlock)
+			{
+		                $aCurrentBlock = array($iRequestCost, $t);
+        		}
+			else
+			{
+				// add RequestCost
+				// remove leak * the time since the last request 
+	        		$aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
+		        	$aCurrentBlock[1] = $t;
+			}
+
+	        	if ($aCurrentBlock[0] <= 0)
+			{
+                		$m->delete($sKey);
+		        }
+			else
+			{
+				// If we have hit the threshold stop and record this to the block list
+				if ($aCurrentBlock[0] >= $iThreshold)
+				{
+					$aCurrentBlock[0] = $iThreshold;
+
+					// Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
+					$i = 10;
+					for($i = 0; $i < 10; $i++)
+					{
+						$aBlockedList = $m->get('blockedList', null, $hCasToken);
+						if (!$aBlockedList)
+						{
+							$aBlockedList = array();
+							$m->add('blockedList', $aBlockedList);
+							$aBlockedList = $m->get('blockedList', null, $hCasToken);
+						}
+						if (!isset($aBlockedList[$sKey]))
+						{
+							$aBlockedList[$sKey] = array(1, $t);
+						}
+						else
+						{
+							$aBlockedList[$sKey][0]++;
+							$aBlockedList[$sKey][1] = $t;
+						}
+						$x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
+						if ($x) break;
+					}
+				}
+				// Only keep in memcache until the time it would have expired (to avoid clutering memcache)
+	                	$m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
+			}
+
+			// Bucket result in the largest bucket we find
+			$iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
+		}
+
+		return $iMaxVal;
+        }
+
+	function getBucketBlocks()
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return null;
+	        $t = time();
+		$aBlockedList = $m->get('blockedList', null, $hCasToken);
+		if (!$aBlockedList) $aBlockedList = array();
+		foreach($aBlockedList as $sKey => $aDetails)
+		{
+			$aCurrentBlock = $m->get($sKey);
+			if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
+			$iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
+			$aBlockedList[$sKey] = array(
+				'totalBlocks' => $aDetails[0],
+				'lastBlockTimestamp' => $aDetails[1],
+				'currentBucketSize' => $iCurrentBucketSize,
+				'lastRequestBlocked' => $aCurrentBlock[0] >= CONST_ConnectionBucket_BlockLimit,
+				'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
+				);
+		}
+		return $aBlockedList;
+	}
+
+	function clearBucketBlocks()
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return false;
+		$m->delete('blockedList');
+		return true;
+	}

settings/settings.php

@@ -14,6 +14,24 @@
 	@define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql');
 	@define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
 
+	// Connection buckets to rate limit people being nasty
+	@define('CONST_ConnectionBucket_MemcacheServerAddress', false);
+	@define('CONST_ConnectionBucket_MemcacheServerPort', 11211);
+	@define('CONST_ConnectionBucket_LeakRate', 1);
+	@define('CONST_ConnectionBucket_BlockLimit', 10);
+	@define('CONST_ConnectionBucket_WaitLimit', 6);
+	@define('CONST_ConnectionBucket_Cost_Reverse', 1);
+	@define('CONST_ConnectionBucket_Cost_Search', 2);
+	@define('CONST_ConnectionBucket_Cost_Details', 3);
+
+	if (!function_exists('user_busy_cost'))
+	{
+		function user_busy_cost()
+		{
+			return 0;
+		}
+	}
+
 	// Website settings
 	@define('CONST_ClosedForIndexing', false);
 	@define('CONST_ClosedForIndexingExceptionIPs', '');
@@ -30,10 +48,11 @@
 
 	@define('CONST_Search_AreaPolygons_Enabled', true);
 	@define('CONST_Search_AreaPolygons', true);
-	@define('CONST_Search_TryDroppedAddressTerms', false);
 
 	@define('CONST_Suggestions_Enabled', false);
 
+	@define('CONST_Search_TryDroppedAddressTerms', false);
+
 	// Set to zero to disable polygon output
 	@define('CONST_PolygonOutput_MaximumTypes', 1);
 

utils/blocks.php

@@ -0,0 +1,40 @@
+#!/usr/bin/php -Cq
+<?php
+
+        require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
+        ini_set('memory_limit', '800M');
+
+	$aCMDOptions = array(
+		"Manage service blocks / restrictions",
+		array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+		array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+		array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+		array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
+		array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),
+	);
+	getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
+
+	$m = getBucketMemcache();
+        if (!$m)
+	{
+		echo "ERROR: Bucket memcache is not configured\n";
+		exit;
+	}
+
+	if ($aResult['list'])
+	{
+		$aBlocks = getBucketBlocks();
+		echo "\n";
+		printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Req Blocked", "Last Block Time");
+		printf(" %'--40s | %'-12s | %'-7s | %'-13s | %'-16s | %'-31s\n", "", "", "", "", "", "");
+		foreach($aBlocks as $sKey => $aDetails)
+		{
+			printf(" %-40s | %12s | %7s | %13s | %16s | %31s\n", $sKey, $aDetails['totalBlocks'], (int)$aDetails['currentBucketSize'], $aDetails['lastRequestBlocked']?'Y':'N', $aDetails['currentlyBlocked']?'Y':'N', date("r", $aDetails['lastBlockTimestamp']));
+		}
+		echo "\n";
+	}
+
+	if ($aResult['delete'])
+	{
+		clearBucketBlocks();
+	}