Merge remote-tracking branch 'upstream/master' into travis-ci

This commit is contained in:
Marc Tobias Metten 2016-07-30 21:53:27 +02:00
commit 8a823d1361
20 changed files with 120 additions and 616 deletions

View File

@ -24,7 +24,7 @@ Now you can install all packages needed for Nominatim:
bzip2-devel proj-devel geos-devel libxml2-devel boost-devel expat-devel zlib-devel
If you want to run the test suite, you need to install the following
aditional packages:
additional packages:
sudo yum install -y python-pip python-Levenshtein python-psycopg2 \
php-phpunit-PHPUnit
@ -83,7 +83,8 @@ Now start the postgresql service after updating this config file.
Finally, we need to add two postgres users: one for the user that does
the import and another for the webserver ro access the database:
the import and another for the webserver which should access the database
only for reading:
sudo -u postgres createuser -s $USERNAME
@ -147,7 +148,7 @@ Get the source code from Github and change into the source directory
The code is built in a special directory. Create this directory,
The code must be built in a separate directory. Create this directory,
then configure and build Nominatim in there:
mkdir build

View File

@ -27,7 +27,7 @@ Now you can install all packages needed for Nominatim:
git
If you want to run the test suite, you need to install the following
aditional packages:
additional packages:
sudo apt-get install -y python-dev python-pip python-levenshtein python-shapely \
python-psycopg2 tidy python-nose python-tidylib \
@ -80,7 +80,8 @@ Restart the postgresql service after updating this config file.
Finally, we need to add two postgres users: one for the user that does
the import and another for the webserver ro access the database:
the import and another for the webserver which should access the database
for reading only:
sudo -u postgres createuser -s $USERNAME
@ -133,7 +134,7 @@ Get the source code from Github and change into the source directory
The code is built in a special directory. Create this directory,
The code must be built in a separate directory. Create this directory,
then configure and build Nominatim in there:
mkdir build

View File

@ -1,5 +1,6 @@
<?php
require_once(CONST_BasePath.'/lib/PlaceLookup.php');
require_once(CONST_BasePath.'/lib/ReverseGeocode.php');
class Geocode
{
@ -62,11 +63,6 @@
$this->aLangPrefOrder = $aLangPref;
}
function setIncludeAddressDetails($bAddressDetails = true)
{
$this->bIncludeAddressDetails = (bool)$bAddressDetails;
}
function getIncludeAddressDetails()
{
return $this->bIncludeAddressDetails;
@ -87,21 +83,11 @@
$this->bIncludePolygonAsPoints = $b;
}
function getIncludePolygonAsPoints()
{
return $this->bIncludePolygonAsPoints;
}
function setIncludePolygonAsText($b = true)
{
$this->bIncludePolygonAsText = $b;
}
function getIncludePolygonAsText()
{
return $this->bIncludePolygonAsText;
}
function setIncludePolygonAsGeoJSON($b = true)
{
$this->bIncludePolygonAsGeoJSON = $b;
@ -122,11 +108,6 @@
$this->fPolygonSimplificationThreshold = $f;
}
function setDeDupe($bDeDupe = true)
{
$this->bDeDupe = (bool)$bDeDupe;
}
function setLimit($iLimit = 10)
{
if ($iLimit > 50) $iLimit = 50;
@ -136,32 +117,11 @@
$this->iLimit = $this->iFinalLimit + min($this->iFinalLimit, 10);
}
function setOffset($iOffset = 0)
{
$this->iOffset = $iOffset;
}
function setFallback($bFallback = true)
{
$this->bFallback = (bool)$bFallback;
}
function setExcludedPlaceIDs($a)
{
// TODO: force to int
$this->aExcludePlaceIDs = $a;
}
function getExcludedPlaceIDs()
{
return $this->aExcludePlaceIDs;
}
function setBounded($bBoundedSearch = true)
{
$this->bBoundedSearch = (bool)$bBoundedSearch;
}
function setViewBox($fLeft, $fBottom, $fRight, $fTop)
{
$this->aViewBox = array($fLeft, $fBottom, $fRight, $fTop);
@ -173,11 +133,6 @@
return $this->aViewBox[0].','.$this->aViewBox[3].','.$this->aViewBox[2].','.$this->aViewBox[1];
}
function setRoute($aRoutePoints)
{
$this->aRoutePoints = $aRoutePoints;
}
function setFeatureType($sFeatureType)
{
switch($sFeatureType)
@ -199,8 +154,8 @@
function setRankRange($iMin, $iMax)
{
$this->iMinAddressRank = (int)$iMin;
$this->iMaxAddressRank = (int)$iMax;
$this->iMinAddressRank = $iMin;
$this->iMaxAddressRank = $iMax;
}
function setNearPoint($aNearPoint, $fRadiusDeg = 0.1)
@ -208,11 +163,6 @@
$this->aNearPoint = array((float)$aNearPoint[0], (float)$aNearPoint[1], (float)$fRadiusDeg);
}
function setCountryCodesList($aCountryCodes)
{
$this->aCountryCodes = $aCountryCodes;
}
function setQuery($sQueryString)
{
$this->sQuery = $sQueryString;
@ -1696,9 +1646,17 @@
else
{
// Just interpret as a reverse geocode
$iPlaceID = geocodeReverse((float)$this->aNearPoint[0], (float)$this->aNearPoint[1]);
if ($iPlaceID)
$aSearchResults = $this->getDetails(array($iPlaceID));
$oReverse = new ReverseGeocode($this->oDB);
$oReverse->setZoom(18);
$aLookup = $oReverse->lookup((float)$this->aNearPoint[0],
(float)$this->aNearPoint[1],
false);
if (CONST_Debug) var_dump("Reverse search", $aLookup);
if ($aLookup['place_id'])
$aSearchResults = $this->getDetails(array($aLookup['place_id'] => -1));
else
$aSearchResults = array();
}
@ -1726,19 +1684,19 @@
if (CONST_Debug) { echo '<i>Recheck words:<\i>'; var_dump($aRecheckWords); }
$oPlaceLookup = new PlaceLookup($this->oDB);
$oPlaceLookup->setIncludePolygonAsPoints($this->bIncludePolygonAsPoints);
$oPlaceLookup->setIncludePolygonAsText($this->bIncludePolygonAsText);
$oPlaceLookup->setIncludePolygonAsGeoJSON($this->bIncludePolygonAsGeoJSON);
$oPlaceLookup->setIncludePolygonAsKML($this->bIncludePolygonAsKML);
$oPlaceLookup->setIncludePolygonAsSVG($this->bIncludePolygonAsSVG);
$oPlaceLookup->setPolygonSimplificationThreshold($this->fPolygonSimplificationThreshold);
foreach($aSearchResults as $iResNum => $aResult)
{
// Default
$fDiameter = getResultDiameter($aResult);
$oPlaceLookup = new PlaceLookup($this->oDB);
$oPlaceLookup->setIncludePolygonAsPoints($this->bIncludePolygonAsPoints);
$oPlaceLookup->setIncludePolygonAsText($this->bIncludePolygonAsText);
$oPlaceLookup->setIncludePolygonAsGeoJSON($this->bIncludePolygonAsGeoJSON);
$oPlaceLookup->setIncludePolygonAsKML($this->bIncludePolygonAsKML);
$oPlaceLookup->setIncludePolygonAsSVG($this->bIncludePolygonAsSVG);
$oPlaceLookup->setPolygonSimplificationThreshold($this->fPolygonSimplificationThreshold);
$aOutlineResult = $oPlaceLookup->getOutlines($aResult['place_id'], $aResult['lon'], $aResult['lat'], $fDiameter/2);
if ($aOutlineResult)
{

View File

@ -3,18 +3,10 @@
{
protected $oDB;
protected $iPlaceID;
protected $sType = false;
protected $fTigerFraction = -1;
protected $aLangPrefOrder = array();
protected $bAddressDetails = false;
protected $bExtraTags = false;
protected $bNameDetails = false;
protected $bIncludePolygonAsPoints = false;
@ -91,38 +83,23 @@
$this->fPolygonSimplificationThreshold = $f;
}
function setPlaceID($iPlaceID)
{
$this->iPlaceID = $iPlaceID;
}
function setOSMID($sType, $iID)
function lookupOSMID($sType, $iID)
{
$sSQL = "select place_id from placex where osm_type = '".pg_escape_string($sType)."' and osm_id = ".(int)$iID." order by type = 'postcode' asc";
$this->iPlaceID = chksql($this->oDB->getOne($sSQL));
$iPlaceID = chksql($this->oDB->getOne($sSQL));
return $this->lookup((int)$iPlaceID);
}
function lookupPlace($details)
function lookup($iPlaceID, $sType = '', $fInterpolFraction = 0.0)
{
if (isset($details['place_id'])) $this->iPlaceID = $details['place_id'];
if (isset($details['type'])) $this->sType = $details['type'];
if (isset($details['osm_type']) && isset($details['osm_id']))
{
$this->setOSMID($details['osm_type'], $details['osm_id']);
}
if (isset($details['fraction'])) $this->fInterpolFraction = $details['fraction'];
return $this->lookup();
}
function lookup()
{
if (!$this->iPlaceID) return null;
if (!$iPlaceID) return null;
$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
$bIsTiger = CONST_Use_US_Tiger_Data && $sType == 'tiger';
$bIsInterpolation = $sType == 'interpolation';
if (CONST_Use_US_Tiger_Data && $this->sType == 'tiger')
if ($bIsTiger)
{
$sSQL = "select place_id,partition, 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, housenumber, null as street, null as isin, postcode,";
$sSQL .= " 'us' as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
@ -134,13 +111,13 @@
if ($this->bNameDetails) $sSQL .= " null as names,";
$sSQL .= " ST_X(point) as lon, ST_Y(point) as lat from (select *, ST_LineInterpolatePoint(linegeo, (housenumber-startnumber::float)/(endnumber-startnumber)::float) as point from ";
$sSQL .= " (select *, ";
$sSQL .= " CASE WHEN interpolationtype='odd' THEN floor((".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2+1";
$sSQL .= " WHEN interpolationtype='even' THEN ((".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2";
$sSQL .= " WHEN interpolationtype='all' THEN (".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)::int";
$sSQL .= " CASE WHEN interpolationtype='odd' THEN floor((".$fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2+1";
$sSQL .= " WHEN interpolationtype='even' THEN ((".$fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2";
$sSQL .= " WHEN interpolationtype='all' THEN (".$fInterpolFraction."*(endnumber-startnumber)+startnumber)::int";
$sSQL .= " END as housenumber";
$sSQL .= " from location_property_tiger where place_id = ".(int)$this->iPlaceID.") as blub1) as blub2";
$sSQL .= " from location_property_tiger where place_id = ".$iPlaceID.") as blub1) as blub2";
}
else if ($this->sType == 'interpolation')
else if ($bIsInterpolation)
{
$sSQL = "select place_id, partition, 'W' as osm_type, osm_id, 'place' as class, 'house' as type, null admin_level, housenumber, null as street, null as isin, postcode,";
$sSQL .= " calculated_country_code as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
@ -152,11 +129,11 @@
if ($this->bNameDetails) $sSQL .= " null as names,";
$sSQL .= " ST_X(point) as lon, ST_Y(point) as lat from (select *, ST_LineInterpolatePoint(linegeo, (housenumber-startnumber::float)/(endnumber-startnumber)::float) as point from ";
$sSQL .= " (select *, ";
$sSQL .= " CASE WHEN interpolationtype='odd' THEN floor((".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2+1";
$sSQL .= " WHEN interpolationtype='even' THEN ((".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2";
$sSQL .= " WHEN interpolationtype='all' THEN (".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)::int";
$sSQL .= " CASE WHEN interpolationtype='odd' THEN floor((".$fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2+1";
$sSQL .= " WHEN interpolationtype='even' THEN ((".$fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2";
$sSQL .= " WHEN interpolationtype='all' THEN (".$fInterpolFraction."*(endnumber-startnumber)+startnumber)::int";
$sSQL .= " END as housenumber";
$sSQL .= " from location_property_osmline where place_id = ".(int)$this->iPlaceID.") as blub1) as blub2";
$sSQL .= " from location_property_osmline where place_id = ".$iPlaceID.") as blub1) as blub2";
// testcase: interpolationtype=odd, startnumber=1000, endnumber=1006, fInterpolFraction=1 => housenumber=1007 => error in st_lineinterpolatepoint
// but this will never happen, because if the searched point is that close to the endnumber, the endnumber house will be directly taken from placex (in ReverseGeocode.php line 220)
// and not interpolated
@ -172,7 +149,7 @@
if ($this->bNameDetails) $sSQL .= " hstore_to_json(name) as names,";
$sSQL .= " (case when centroid is null then st_y(st_centroid(geometry)) else st_y(centroid) end) as lat,";
$sSQL .= " (case when centroid is null then st_x(st_centroid(geometry)) else st_x(centroid) end) as lon";
$sSQL .= " from placex where place_id = ".(int)$this->iPlaceID;
$sSQL .= " from placex where place_id = ".$iPlaceID;
}
$aPlace = chksql($this->oDB->getRow($sSQL), "Could not lookup place");
@ -181,11 +158,10 @@
if ($this->bAddressDetails)
{
if(CONST_Use_US_Tiger_Data && $this->sType == 'tiger' || $this->sType == 'interpolation') // to get addressdetails for tiger data, the housenumber is needed
$aAddress = $this->getAddressNames($aPlace['housenumber']);
else
$aAddress = $this->getAddressNames();
$aPlace['aAddress'] = $aAddress;
// to get addressdetails for tiger data, the housenumber is needed
$iHousenumber = ($bIsTiger || $bIsInterpolation) ? $aPlace['housenumber'] : -1;
$aPlace['aAddress'] = $this->getAddressNames($aPlace['place_id'],
$iHousenumber);
}
if ($this->bExtraTags)
@ -232,22 +208,20 @@
return $aPlace;
}
function getAddressDetails($bAll = false, $housenumber = -1)
function getAddressDetails($iPlaceID, $bAll = false, $housenumber = -1)
{
if (!$this->iPlaceID) return null;
$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
$sSQL = "select *,get_name_by_language(name,$sLanguagePrefArraySQL) as localname from get_addressdata(".$this->iPlaceID.",".$housenumber.")";
$sSQL = "select *,get_name_by_language(name,$sLanguagePrefArraySQL) as localname from get_addressdata(".$iPlaceID.",".$housenumber.")";
if (!$bAll) $sSQL .= " WHERE isaddress OR type = 'country_code'";
$sSQL .= " order by rank_address desc,isaddress desc";
return chksql($this->oDB->getAll($sSQL));
}
function getAddressNames($housenumber = -1)
function getAddressNames($iPlaceID, $housenumber = -1)
{
$aAddressLines = $this->getAddressDetails(false, $housenumber);
$aAddressLines = $this->getAddressDetails($iPlaceID, false, $housenumber);
$aAddress = array();
$aFallback = array();

View File

@ -2,42 +2,13 @@
class ReverseGeocode
{
protected $oDB;
protected $fLat;
protected $fLon;
protected $iMaxRank = 28;
protected $aLangPrefOrder = array();
protected $bIncludePolygonAsPoints = false;
protected $bIncludePolygonAsText = false;
protected $bIncludePolygonAsGeoJSON = false;
protected $bIncludePolygonAsKML = false;
protected $bIncludePolygonAsSVG = false;
protected $fPolygonSimplificationThreshold = 0.0;
function ReverseGeocode(&$oDB)
{
$this->oDB =& $oDB;
}
function setLanguagePreference($aLangPref)
{
$this->aLangPrefOrder = $aLangPref;
}
function setLatLon($fLat, $fLon)
{
$this->fLat = (float)$fLat;
$this->fLon = (float)$fLon;
}
function setRank($iRank)
{
$this->iMaxRank = $iRank;
}
function setZoom($iZoom)
{
// Zoom to rank, this could probably be calculated but a lookup gives fine control
@ -66,53 +37,12 @@
$this->iMaxRank = (isset($iZoom) && isset($aZoomRank[$iZoom]))?$aZoomRank[$iZoom]:28;
}
function setIncludePolygonAsPoints($b = true)
{
$this->bIncludePolygonAsPoints = $b;
}
function getIncludePolygonAsPoints()
{
return $this->bIncludePolygonAsPoints;
}
function setIncludePolygonAsText($b = true)
{
$this->bIncludePolygonAsText = $b;
}
function getIncludePolygonAsText()
{
return $this->bIncludePolygonAsText;
}
function setIncludePolygonAsGeoJSON($b = true)
{
$this->bIncludePolygonAsGeoJSON = $b;
}
function setIncludePolygonAsKML($b = true)
{
$this->bIncludePolygonAsKML = $b;
}
function setIncludePolygonAsSVG($b = true)
{
$this->bIncludePolygonAsSVG = $b;
}
function setPolygonSimplificationThreshold($f)
{
$this->fPolygonSimplificationThreshold = $f;
}
// returns { place_id =>, type => '(osm|tiger)' }
// fails if no place was found
function lookup()
function lookup($fLat, $fLon, $bDoInterpolation = true)
{
$sPointSQL = 'ST_SetSRID(ST_Point('.$this->fLon.','.$this->fLat.'),4326)';
$sPointSQL = 'ST_SetSRID(ST_Point('.$fLon.','.$fLat.'),4326)';
$iMaxRank = $this->iMaxRank;
$iMaxRank_orig = $this->iMaxRank;
// Find the nearest point
$fSearchDiam = 0.0004;
@ -155,7 +85,7 @@
$bIsInUnitedStates = ($aPlace['calculated_country_code'] == 'us');
}
// if a street or house was found, look in interpolation lines table
if ($iMaxRank_orig >= 28 && $aPlace && $aPlace['rank_search'] >= 26)
if ($bDoInterpolation && $this->iMaxRank >= 28 && $aPlace && $aPlace['rank_search'] >= 26)
{
// if a house was found, search the interpolation line that is at least as close as the house
$sSQL = 'SELECT place_id, parent_place_id, 30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction';
@ -220,7 +150,7 @@
}
// Only street found? If it's in the US we can check TIGER data for nearest housenumber
if (CONST_Use_US_Tiger_Data && $bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
if (CONST_Use_US_Tiger_Data && $bDoInterpolation && $bIsInUnitedStates && $this->iMaxRank >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
{
$fSearchDiam = 0.001;
$sSQL = 'SELECT place_id,parent_place_id,30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction';

View File

@ -13,47 +13,5 @@
}
if ($_SERVER['REQUEST_METHOD'] == 'OPTIONS') exit;
$aBucketKeys = array();
if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST)));
if (isset($_SERVER["REMOTE_ADDR"])) $aBucketKeys[] = $_SERVER["REMOTE_ADDR"];
if (isset($_GET["email"])) $aBucketKeys[] = $_GET["email"];
$fBucketVal = doBucket($aBucketKeys,
(defined('CONST_ConnectionBucket_PageType')?constant('CONST_ConnectionBucket_Cost_'.CONST_ConnectionBucket_PageType):1) + user_busy_cost(),
CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
if ($fBucketVal > CONST_ConnectionBucket_WaitLimit && $fBucketVal < CONST_ConnectionBucket_BlockLimit)
{
$m = getBucketMemcache();
$iCurrentSleeping = $m->increment('sleepCounter');
if (false === $iCurrentSleeping)
{
$m->add('sleepCounter', 0);
$iCurrentSleeping = $m->increment('sleepCounter');
}
if ($iCurrentSleeping >= CONST_ConnectionBucket_MaxSleeping || isBucketSleeping($aBucketKeys))
{
// Too many threads sleeping already. This becomes a hard block.
$fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_BlockLimit, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
}
else
{
setBucketSleeping($aBucketKeys, true);
sleep(($fBucketVal - CONST_ConnectionBucket_WaitLimit)/CONST_ConnectionBucket_LeakRate);
$fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
setBucketSleeping($aBucketKeys, false);
}
$m->decrement('sleepCounter');
}
if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit)
{
header("HTTP/1.0 429 Too Many Requests");
echo "Your IP has been blocked. \n";
echo CONST_BlockMessage;
exit;
}
header('Content-type: text/html; charset=utf-8');

View File

@ -1,7 +1,6 @@
<?php
require_once(CONST_BasePath.'/lib/lib.php');
require_once(CONST_BasePath.'/lib/leakybucket.php');
require_once(CONST_BasePath.'/lib/db.php');
if (get_magic_quotes_gpc())

View File

@ -1,168 +0,0 @@
<?php
function getBucketMemcache()
{
static $m;
if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
if (!isset($m))
{
$m = new Memcached();
$m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
}
return $m;
}
function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
{
$m = getBucketMemcache();
if (!$m) return 0;
$iMaxVal = 0;
$t = time();
foreach($asKey as $sKey)
{
$aCurrentBlock = $m->get($sKey);
if (!$aCurrentBlock)
{
$aCurrentBlock = array($iRequestCost, $t, false);
}
else
{
// add RequestCost
// remove leak * the time since the last request
$aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
$aCurrentBlock[1] = $t;
}
if ($aCurrentBlock[0] <= 0)
{
$m->delete($sKey);
}
else
{
// If we have hit the threshold stop and record this to the block list
if ($aCurrentBlock[0] >= $iThreshold)
{
$aCurrentBlock[0] = $iThreshold;
// Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
$i = 10;
for($i = 0; $i < 10; $i++)
{
$aBlockedList = $m->get('blockedList', null, $hCasToken);
if (!$aBlockedList)
{
$aBlockedList = array();
$m->add('blockedList', $aBlockedList);
$aBlockedList = $m->get('blockedList', null, $hCasToken);
}
if (!isset($aBlockedList[$sKey]))
{
$aBlockedList[$sKey] = array(1, $t);
}
else
{
$aBlockedList[$sKey][0]++;
$aBlockedList[$sKey][1] = $t;
}
if (sizeof($aBlockedList) > CONST_ConnectionBucket_MaxBlockList)
{
uasort($aBlockedList, 'byValue1');
$aBlockedList = array_slice($aBlockedList, 0, CONST_ConnectionBucket_MaxBlockList);
}
$x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
if ($x) break;
}
}
// Only keep in memcache until the time it would have expired (to avoid clutering memcache)
$m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
}
// Bucket result in the largest bucket we find
$iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
}
return $iMaxVal;
}
function isBucketSleeping($asKey)
{
$m = getBucketMemcache();
if (!$m) return false;
foreach($asKey as $sKey)
{
$aCurrentBlock = $m->get($sKey);
if ($aCurrentBlock[2]) return true;
}
return false;
}
function setBucketSleeping($asKey, $bVal)
{
$m = getBucketMemcache();
if (!$m) return false;
$iMaxVal = 0;
$t = time();
foreach($asKey as $sKey)
{
$aCurrentBlock = $m->get($sKey);
$aCurrentBlock[2] = $bVal;
$m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/CONST_ConnectionBucket_LeakRate);
}
return true;
}
function byValue1($a, $b)
{
if ($a[1] == $b[1])
{
return 0;
}
return ($a[1] > $b[1]) ? -1 : 1;
}
function byLastBlockTime($a, $b)
{
if ($a['lastBlockTimestamp'] == $b['lastBlockTimestamp'])
{
return 0;
}
return ($a['lastBlockTimestamp'] > $b['lastBlockTimestamp']) ? -1 : 1;
}
function getBucketBlocks()
{
$m = getBucketMemcache();
if (!$m) return null;
$t = time();
$aBlockedList = $m->get('blockedList', null, $hCasToken);
if (!$aBlockedList) $aBlockedList = array();
foreach($aBlockedList as $sKey => $aDetails)
{
$aCurrentBlock = $m->get($sKey);
if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
$iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
$aBlockedList[$sKey] = array(
'totalBlocks' => $aDetails[0],
'lastBlockTimestamp' => $aDetails[1],
'isSleeping' => (isset($aCurrentBlock[2])?$aCurrentBlock[2]:false),
'currentBucketSize' => $iCurrentBucketSize,
'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
);
}
uasort($aBlockedList, 'byLastBlockTime');
return $aBlockedList;
}
function clearBucketBlocks()
{
$m = getBucketMemcache();
if (!$m) return false;
$m->delete('blockedList');
return true;
}

View File

@ -9,14 +9,6 @@
}
function getLoadAverage()
{
$sLoadAverage = file_get_contents('/proc/loadavg');
$aLoadAverage = explode(' ',$sLoadAverage);
return (float)$aLoadAverage[0];
}
function getProcessorCount()
{
$sCPU = file_get_contents('/proc/cpuinfo');
@ -688,90 +680,6 @@
}
function geocodeReverse($fLat, $fLon, $iZoom=18)
{
$oDB =& getDB();
$sPointSQL = "ST_SetSRID(ST_Point($fLon,$fLat),4326)";
// Zoom to rank, this could probably be calculated but a lookup gives fine control
$aZoomRank = array(
0 => 2, // Continent / Sea
1 => 2,
2 => 2,
3 => 4, // Country
4 => 4,
5 => 8, // State
6 => 10, // Region
7 => 10,
8 => 12, // County
9 => 12,
10 => 17, // City
11 => 17,
12 => 18, // Town / Village
13 => 18,
14 => 22, // Suburb
15 => 22,
16 => 26, // Street, TODO: major street?
17 => 26,
18 => 30, // or >, Building
19 => 30, // or >, Building
);
$iMaxRank = isset($aZoomRank[$iZoom])?$aZoomRank[$iZoom]:28;
// Find the nearest point
$fSearchDiam = 0.0001;
$iPlaceID = null;
$aArea = false;
$fMaxAreaDistance = 1;
while(!$iPlaceID && $fSearchDiam < $fMaxAreaDistance)
{
$fSearchDiam = $fSearchDiam * 2;
// If we have to expand the search area by a large amount then we need a larger feature
// then there is a limit to how small the feature should be
if ($fSearchDiam > 2 && $iMaxRank > 4) $iMaxRank = 4;
if ($fSearchDiam > 1 && $iMaxRank > 9) $iMaxRank = 8;
if ($fSearchDiam > 0.8 && $iMaxRank > 10) $iMaxRank = 10;
if ($fSearchDiam > 0.6 && $iMaxRank > 12) $iMaxRank = 12;
if ($fSearchDiam > 0.2 && $iMaxRank > 17) $iMaxRank = 17;
if ($fSearchDiam > 0.1 && $iMaxRank > 18) $iMaxRank = 18;
if ($fSearchDiam > 0.008 && $iMaxRank > 22) $iMaxRank = 22;
if ($fSearchDiam > 0.001 && $iMaxRank > 26) $iMaxRank = 26;
$sSQL = 'select place_id,parent_place_id from placex';
$sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')';
$sSQL .= ' and rank_search != 28 and rank_search >= '.$iMaxRank;
$sSQL .= ' and (name is not null or housenumber is not null)';
$sSQL .= ' and class not in (\'waterway\')';
$sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') ';
$sSQL .= ' OR ST_DWithin('.$sPointSQL.', ST_Centroid(geometry), '.$fSearchDiam.'))';
$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1';
//var_dump($sSQL);
$aPlace = chksql($oDB->getRow($sSQL));
$iPlaceID = $aPlace['place_id'];
}
// The point we found might be too small - use the address to find what it is a child of
if ($iPlaceID)
{
$sSQL = "select address_place_id from place_addressline where cached_rank_address <= $iMaxRank and place_id = $iPlaceID order by cached_rank_address desc,isaddress desc,distance desc limit 1";
$iPlaceID = chksql($oDB->getOne($sSQL));
if ($iPlaceID && $aPlace['place_id'] && $iMaxRank < 28)
{
$sSQL = "select address_place_id from place_addressline where cached_rank_address <= $iMaxRank and place_id = ".$aPlace['place_id']." order by cached_rank_address desc,isaddress desc,distance desc";
$iPlaceID = chksql($oDB->getOne($sSQL));
}
if (!$iPlaceID)
{
$iPlaceID = $aPlace['place_id'];
}
}
return $iPlaceID;
}
function addQuotes($s)
{
return "'".$s."'";

View File

@ -62,34 +62,8 @@
@define('CONST_Replication_Update_Interval', '60'); // How often upstream publishes diffs
@define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet
// Connection buckets to rate limit people being nasty
@define('CONST_ConnectionBucket_MemcacheServerAddress', false);
@define('CONST_ConnectionBucket_MemcacheServerPort', 11211);
@define('CONST_ConnectionBucket_MaxBlockList', 100);
@define('CONST_ConnectionBucket_LeakRate', 1);
@define('CONST_ConnectionBucket_BlockLimit', 10);
@define('CONST_ConnectionBucket_WaitLimit', 6);
@define('CONST_ConnectionBucket_MaxSleeping', 10);
@define('CONST_ConnectionBucket_Cost_Reverse', 1);
@define('CONST_ConnectionBucket_Cost_Search', 2);
@define('CONST_ConnectionBucket_Cost_Details', 3);
@define('CONST_ConnectionBucket_Cost_Status', 1);
// Override this function to add an adjustment factor to the cost
// based on server load. e.g. getBlockingProcesses
if (!function_exists('user_busy_cost'))
{
function user_busy_cost()
{
return 0;
}
}
// Website settings
@define('CONST_NoAccessControl', true);
@define('CONST_BlockedIPs', '');
@define('CONST_BulkUserIPs', '');
@define('CONST_BlockMessage', ''); // additional info to show for blocked IPs
@define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/');
// Language to assume when none is supplied with the query.

View File

@ -18,9 +18,9 @@ Feature: Reverse geocoding
| xml | 4
When looking up coordinates 53.9788769,13.0830313
And results contain valid boundingboxes
Scenario: Reverse geocoding for odd interpolated housenumber
Scenario: Reverse geocoding for even interpolated housenumber
@Tiger
@ -32,9 +32,10 @@ Feature: Reverse geocoding
And exactly 1 result is returned
And result addresses contain
| ID | house_number | road | postcode | country_code
| 0 | 7096 | Kings Estate Drive | 84128 | us
And result 0 has not attributes osm_id,osm_type
| 0 | 709. | Kings Estate Drive | 84128 | us
And results contain
| osm_type
| tiger
@Tiger
Scenario: No TIGER house number for zoom < 18

View File

@ -761,6 +761,7 @@
function pgsqlRunPartitionScript($sTemplate)
{
global $aCMDResult;
$oDB =& getDB();
$sSQL = 'select distinct partition from country_name';

View File

@ -27,7 +27,10 @@
if (!$aResult['search-only']) {
$oReverseGeocode = new ReverseGeocode($oDB);
$oReverseGeocode->setIncludeAddressDetails(true);
$oReverseGeocode->setZoom(20);
$oPlaceLookup = new PlaceLookup($oDB);
$oPlaceLookup->setIncludeAddressDetails(true);
$oPlaceLookup->setLanguagePreference(array('en'));
echo "Warm reverse: ";
if ($bVerbose) echo "\n";
@ -35,11 +38,13 @@
$fLat = rand(-9000, 9000) / 100;
$fLon = rand(-18000, 18000) / 100;
if ($bVerbose) echo "$fLat, $fLon = ";
$oReverseGeocode->setLanguagePreference(array('en'));
$oReverseGeocode->setLatLon($fLat, $fLon);
$oReverseGeocode->setZoom(20);
$aDetails = $oReverseGeocode->lookup();
if ($bVerbose) echo $aDetails['langaddress']."\n";
$aLookup = $oReverseGeocode->lookup($fLat, $fLon);
if ($aLookup && $aLookup['place_id'])
{
$aDetails = $oPlaceLookup->lookup((int)$aLookup['place_id'],
$aLookup['type'], $aLookup['fraction']);
if ($bVerbose) echo $aDetails['langaddress']."\n";
}
else echo ".";
}
echo "\n";

View File

@ -25,7 +25,7 @@
bzip2-devel proj-devel geos-devel libxml2-devel boost-devel expat-devel zlib-devel
# If you want to run the test suite, you need to install the following
# aditional packages:
# additional packages:
sudo yum install -y python-pip python-Levenshtein python-psycopg2 \
php-phpunit-PHPUnit
@ -84,7 +84,8 @@
#
# Finally, we need to add two postgres users: one for the user that does
# the import and another for the webserver ro access the database:
# the import and another for the webserver which should access the database
# only for reading:
#
sudo -u postgres createuser -s $USERNAME
@ -148,7 +149,7 @@ else #DOCS:
cd $USERHOME #DOCS:
fi #DOCS:
# The code is built in a special directory. Create this directory,
# The code must be built in a separate directory. Create this directory,
# then configure and build Nominatim in there:
mkdir build

View File

@ -28,7 +28,7 @@ sudo update-locale LANG=en_US.UTF-8 #DOCS:
git
# If you want to run the test suite, you need to install the following
# aditional packages:
# additional packages:
sudo apt-get install -y python-dev python-pip python-levenshtein python-shapely \
python-psycopg2 tidy python-nose python-tidylib \
@ -81,7 +81,8 @@ sudo update-locale LANG=en_US.UTF-8 #DOCS:
#
# Finally, we need to add two postgres users: one for the user that does
# the import and another for the webserver ro access the database:
# the import and another for the webserver which should access the database
# for reading only:
#
sudo -u postgres createuser -s $USERNAME
@ -134,7 +135,7 @@ else #DOCS:
cd $USERHOME #DOCS:
fi #DOCS:
# The code is built in a special directory. Create this directory,
# The code must be built in a separate directory. Create this directory,
# then configure and build Nominatim in there:
mkdir build

View File

@ -7,14 +7,7 @@
require_once(CONST_BasePath.'/lib/output.php');
$sOutputFormat = 'html';
/*
$fLoadAvg = getLoadAverage();
if ($fLoadAvg > 3)
{
echo "Page temporarily blocked due to high server load\n";
exit;
}
*/
ini_set('memory_limit', '200M');
$oDB =& getDB();

View File

@ -57,9 +57,8 @@
$oPlaceLookup = new PlaceLookup($oDB);
$oPlaceLookup->setLanguagePreference($aLangPrefOrder);
$oPlaceLookup->setIncludeAddressDetails(true);
$oPlaceLookup->setPlaceId($iPlaceID);
$aPlaceAddress = array_reverse($oPlaceLookup->getAddressDetails());
$aPlaceAddress = array_reverse($oPlaceLookup->getAddressDetails($iPlaceID));
if (!sizeof($aPlaceAddress)) userError("Unknown place id.");

View File

@ -7,18 +7,6 @@
require_once(CONST_BasePath.'/lib/PlaceLookup.php');
require_once(CONST_BasePath.'/lib/output.php');
if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
{
$fLoadAvg = getLoadAverage();
if ($fLoadAvg > 2) sleep(60);
if ($fLoadAvg > 4) sleep(120);
if ($fLoadAvg > 6)
{
userError("Bulk User: Temporary block due to high server load");
exit;
}
}
$oDB =& getDB();
ini_set('memory_limit', '200M');
@ -56,8 +44,7 @@
if ( $iId > 0 && ($sType == 'N' || $sType == 'W' || $sType == 'R') )
{
$aCleanedQueryParts[] = $sType . $iId;
$oPlaceLookup->setOSMID($sType, $iId);
$oPlace = $oPlaceLookup->lookup();
$oPlace = $oPlaceLookup->lookupOSMID($sType, $iId);
if ($oPlace){
// we want to use the search-* output templates, so we need to fill
// $aSearchResults and slightly change the (reverse search) oPlace

View File

@ -8,19 +8,6 @@
require_once(CONST_BasePath.'/lib/ReverseGeocode.php');
require_once(CONST_BasePath.'/lib/output.php');
if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
{
$fLoadAvg = getLoadAverage();
if ($fLoadAvg > 2) sleep(60);
if ($fLoadAvg > 4) sleep(120);
if ($fLoadAvg > 6)
{
echo "Bulk User: Temporary block due to high server load\n";
exit;
}
}
$bAsGeoJSON = getParamBool('polygon_geojson');
$bAsKML = getParamBool('polygon_kml');
$bAsSVG = getParamBool('polygon_svg');
@ -56,40 +43,38 @@
$hLog = logStart($oDB, 'reverse', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
$oPlaceLookup = new PlaceLookup($oDB);
$oPlaceLookup->setLanguagePreference($aLangPrefOrder);
$oPlaceLookup->setIncludeAddressDetails(getParamBool('addressdetails', true));
$oPlaceLookup->setIncludeExtraTags(getParamBool('extratags', false));
$oPlaceLookup->setIncludeNameDetails(getParamBool('namedetails', false));
$sOsmType = getParamSet('osm_type', array('N', 'W', 'R'));
$iOsmId = getParamInt('osm_id', -1);
$fLat = getParamFloat('lat');
$fLon = getParamFloat('lon');
if ($sOsmType && $iOsmId > 0)
{
$aLookup = array('osm_type' => $sOsmType, 'osm_id' => $iOsmId);
$aPlace = $oPlaceLookup->lookupOSMID($sOsmType, $iOsmId);
}
else if ($fLat !== false && $fLon !==false)
else if ($fLat !== false && $fLon !== false)
{
$oReverseGeocode = new ReverseGeocode($oDB);
$oReverseGeocode->setLanguagePreference($aLangPrefOrder);
$oReverseGeocode->setLatLon($fLat, $fLon);
$oReverseGeocode->setZoom(getParamInt('zoom', 18));
$aLookup = $oReverseGeocode->lookup();
$aLookup = $oReverseGeocode->lookup($fLat, $fLon);
if (CONST_Debug) var_dump($aLookup);
$aPlace = $oPlaceLookup->lookup((int)$aLookup['place_id'],
$aLookup['type'], $aLookup['fraction']);
}
else if ($sOutputFormat != 'html')
{
userError("Need coordinates or OSM object to lookup.");
}
if ($aLookup)
if ($aPlace)
{
$oPlaceLookup = new PlaceLookup($oDB);
$oPlaceLookup->setLanguagePreference($aLangPrefOrder);
$oPlaceLookup->setIncludeAddressDetails(getParamBool('addressdetails', true));
$oPlaceLookup->setIncludeExtraTags(getParamBool('extratags', false));
$oPlaceLookup->setIncludeNameDetails(getParamBool('namedetails', false));
$aPlace = $oPlaceLookup->lookupPlace($aLookup);
$oPlaceLookup->setIncludePolygonAsPoints(false);
$oPlaceLookup->setIncludePolygonAsText($bAsText);
$oPlaceLookup->setIncludePolygonAsGeoJSON($bAsGeoJSON);
@ -98,17 +83,15 @@
$oPlaceLookup->setPolygonSimplificationThreshold($fThreshold);
$fRadius = $fDiameter = getResultDiameter($aPlace);
$aOutlineResult = $oPlaceLookup->getOutlines($aPlace['place_id'], $aPlace['lon'], $aPlace['lat'], $fRadius);
$aOutlineResult = $oPlaceLookup->getOutlines($aPlace['place_id'],
$aPlace['lon'], $aPlace['lat'],
$fRadius);
if ($aOutlineResult)
{
$aPlace = array_merge($aPlace, $aOutlineResult);
}
}
else
{
$aPlace = null;
}
if (CONST_Debug)

View File

@ -32,6 +32,7 @@
if ($sOutputFormat == 'html')
{
$oGeocode->setIncludePolygonAsText(getParamBool('polygon'));
$bAsText = false;
}
else
{
@ -84,22 +85,20 @@
include(CONST_BasePath.'/lib/template/search-batch-json.php');
exit;
}
if (!getParamString('q') && isset($_SERVER['PATH_INFO']) && $_SERVER['PATH_INFO'][0] == '/')
{
$sQuery = substr(rawurldecode($_SERVER['PATH_INFO']), 1);
// reverse order of '/' separated string
$aPhrases = explode('/', $sQuery);
$aPhrases = array_reverse($aPhrases);
$sQuery = join(', ',$aPhrases);
$oGeocode->setQuery($sQuery);
}
else
{
if (!getParamString('q') && isset($_SERVER['PATH_INFO']) && $_SERVER['PATH_INFO'][0] == '/')
{
$sQuery = substr(rawurldecode($_SERVER['PATH_INFO']), 1);
// reverse order of '/' separated string
$aPhrases = explode('/', $sQuery);
$aPhrases = array_reverse($aPhrases);
$sQuery = join(', ',$aPhrases);
$oGeocode->setQuery($sQuery);
}
else
{
$oGeocode->setQueryFromParams($_GET);
}
$oGeocode->setQueryFromParams($_GET);
}
$hLog = logStart($oDB, 'search', $oGeocode->getQueryString(), $aLangPrefOrder);
@ -113,13 +112,12 @@
}
logEnd($oDB, $hLog, sizeof($aSearchResults));
$bAsText = $oGeocode->getIncludePolygonAsText();
$sQuery = $oGeocode->getQueryString();
$sViewBox = $oGeocode->getViewBoxString();
$bShowPolygons = (isset($_GET['polygon']) && $_GET['polygon']);
$aExcludePlaceIDs = $oGeocode->getExcludedPlaceIDs();
$sMoreURL = CONST_Website_BaseURL.'search.php?format='.urlencode($sOutputFormat).'&exclude_place_ids='.join(',',$oGeocode->getExcludedPlaceIDs());
$sMoreURL = CONST_Website_BaseURL.'search.php?format='.urlencode($sOutputFormat).'&exclude_place_ids='.join(',',$aExcludePlaceIDs);
if (isset($_SERVER["HTTP_ACCEPT_LANGUAGE"])) $sMoreURL .= '&accept-language='.$_SERVER["HTTP_ACCEPT_LANGUAGE"];
if ($bShowPolygons) $sMoreURL .= '&polygon=1';
if ($oGeocode->getIncludeAddressDetails()) $sMoreURL .= '&addressdetails=1';