Merge pull request #989 from lonvia/pretty-debug

nicer formatting for Geocode debug output
This commit is contained in:
Sarah Hoffmann 2018-03-26 20:56:57 +02:00 committed by GitHub
commit 64fa70ac0a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 305 additions and 48 deletions

159
lib/DebugHtml.php Normal file
View File

@ -0,0 +1,159 @@
<?php
namespace Nominatim;
class Debug
{
public static function newFunction($sHeading)
{
echo "<pre><h2>Debug output for $sHeading</h2></pre>\n";
}
public static function newSection($sHeading)
{
echo "<hr><pre><h3>$sHeading</h3></pre>\n";
}
public static function printVar($sHeading, $mVar)
{
echo '<pre><b>'.$sHeading. ':</b> ';
Debug::outputVar($mVar, str_repeat(' ', strlen($sHeading) + 3));
echo "</pre>\n";
}
public static function fmtArrayVals($aArr)
{
return array('__debug_format' => 'array_vals', 'data' => $aArr);
}
public static function printDebugArray($sHeading, $oVar)
{
if ($oVar === null) {
Debug::printVar($sHeading, 'null');
} else {
Debug::printVar($sHeading, $oVar->debugInfo());
}
}
public static function printDebugTable($sHeading, $aVar)
{
echo '<b>'.$sHeading.":</b>\n";
echo '<table border="1">';
if (!empty($aVar)) {
echo '<tr>';
$aKeys = array();
$aInfo = reset($aVar);
if (!is_array($aInfo)) {
$aInfo = $aInfo->debugInfo();
}
foreach ($aInfo as $sKey => $mVal) {
echo '<th><small>'.$sKey.'</small></th>';
$aKeys[] = $sKey;
}
echo '</tr>';
foreach ($aVar as $oRow) {
$aInfo = $oRow;
if (!is_array($oRow)) {
$aInfo = $oRow->debugInfo();
}
echo '<tr>';
foreach ($aKeys as $sKey) {
echo '<td><pre>';
if (isset($aInfo[$sKey])) {
Debug::outputVar($aInfo[$sKey], '');
}
echo '</pre></td>';
}
echo '<tr>';
}
}
echo '</table>';
}
public static function printGroupTable($sHeading, $aVar)
{
echo '<b>'.$sHeading.":</b>\n";
echo '<table border="1">';
if (!empty($aVar)) {
echo '<tr><th><small>Group</small></th>';
$aKeys = array();
$aInfo = reset(reset($aVar));
if (!is_array($aInfo)) {
$aInfo = $aInfo->debugInfo();
}
foreach ($aInfo as $sKey => $mVal) {
echo '<th><small>'.$sKey.'</small></th>';
$aKeys[] = $sKey;
}
echo '</tr>';
foreach ($aVar as $sGrpKey => $aGroup) {
foreach ($aGroup as $oRow) {
$aInfo = $oRow;
if (!is_array($oRow)) {
$aInfo = $oRow->debugInfo();
}
echo '<tr><td><pre>'.$sGrpKey.'</pre></td>';
foreach ($aKeys as $sKey) {
echo '<td><pre>';
if (!empty($aInfo[$sKey])) {
Debug::outputVar($aInfo[$sKey], '');
}
echo '</pre></td>';
}
echo '<tr>';
}
}
}
echo '</table>';
}
public static function printSQL($sSQL)
{
echo '<p><tt><font color="#aaa">'.$sSQL.'</font></tt></p>'."\n";
}
private static function outputVar($mVar, $sPreNL)
{
if (is_array($mVar) && !isset($mVar['__debug_format'])) {
$sPre = '';
foreach ($mVar as $mKey => $aValue) {
echo $sPre;
$iKeyLen = Debug::outputSimpleVar($mKey);
echo ' => ';
Debug::outputVar(
$aValue,
$sPreNL.str_repeat(' ', $iKeyLen + 4)
);
$sPre = "\n".$sPreNL;
}
} elseif (is_array($mVar) && isset($mVar['__debug_format'])) {
if (!empty($mVar[data])) {
$sPre = '';
foreach ($mVar[data] as $mValue) {
echo $sPre;
Debug::outputSimpleVar($mValue);
$sPre = ', ';
}
}
} else {
Debug::outputSimpleVar($mVar);
}
}
private static function outputSimpleVar($mVar)
{
if (is_bool($mVar)) {
echo '<i>'.($mVar ? 'True' : 'False').'</i>';
return $mVar ? 4 : 5;
}
if (is_string($mVar)) {
echo "'$mVar'";
return strlen($mVar) + 2;
}
echo (string)$mVar;
return strlen((string)$mVar);
}
}

11
lib/DebugNone.php Normal file
View File

@ -0,0 +1,11 @@
<?php
namespace Nominatim;
class Debug
{
public static function __callStatic($name, $arguments)
{
// nothing
}
}

View File

@ -446,8 +446,6 @@ class Geocode
$aSearches = array_merge($aSearches, $aNewSearches);
if ($iSearchCount > 50) break;
}
//if (CONST_Debug) _debugDumpGroupedSearches($aGroupedSearches, $aValidTokens);
}
// Revisit searches, drop bad searches and give penalty to unlikely combinations.
@ -502,8 +500,11 @@ class Geocode
public function lookup()
{
Debug::newFunction('Geocode::lookup');
if (!$this->sQuery && !$this->aStructuredQuery) return array();
Debug::printDebugArray('Geocode', $this);
$oCtx = new SearchContext();
if ($this->aRoutePoints) {
@ -523,7 +524,11 @@ class Geocode
$oCtx->setCountryList($this->aCountryCodes);
}
Debug::newSection('Query Preprocessing');
$sNormQuery = $this->normTerm($this->sQuery);
Debug::printVar('Normalized query', $sNormQuery);
$sLanguagePrefArraySQL = getArraySQL(
array_map('getDBQuoted', $this->aLangPrefOrder)
);
@ -560,6 +565,10 @@ class Geocode
$aSpecialTermsRaw,
PREG_SET_ORDER
);
if (!empty($aSpecialTermsRaw)) {
Debug::printVar('Special terms', $aSpecialTermsRaw);
}
foreach ($aSpecialTermsRaw as $aSpecialTerm) {
$sQuery = str_replace($aSpecialTerm[0], ' ', $sQuery);
if (!$sSpecialTerm) {
@ -582,7 +591,8 @@ class Geocode
$sSQL = 'SELECT class, type FROM word ';
$sSQL .= ' WHERE word_token in (\' '.$sToken.'\')';
$sSQL .= ' AND class is not null AND class not in (\'place\')';
if (CONST_Debug) var_Dump($sSQL);
Debug::printSQL($sSQL);
$aSearchWords = chksql($this->oDB->getAll($sSQL));
$aNewSearches = array();
foreach ($aSearches as $oSearch) {
@ -609,10 +619,15 @@ class Geocode
$bStructuredPhrases = false;
}
Debug::printDebugArray('Search context', $oCtx);
Debug::printDebugArray('Base search', $aSearches[0]);
Debug::printVar('Final query phrases', $aInPhrases);
// Convert each phrase to standard form
// Create a list of standard words
// Get all 'sets' of words
// Generate a complete list of all
Debug::newSection('Tokenization');
$aTokens = array();
$aPhrases = array();
foreach ($aInPhrases as $iPhrase => $sPhrase) {
@ -627,13 +642,16 @@ class Geocode
}
}
Debug::printDebugTable('Phrases', $aPhrases);
Debug::printVar('Tokens', $aTokens);
if (!empty($aTokens)) {
// Check which tokens we have, get the ID numbers
$sSQL = 'SELECT word_id, word_token, word, class, type, country_code, operator, search_name_count';
$sSQL .= ' FROM word ';
$sSQL .= ' WHERE word_token in ('.join(',', array_map('getDBQuoted', $aTokens)).')';
if (CONST_Debug) var_Dump($sSQL);
Debug::printSQL($sSQL);
$aValidTokens = array();
$aDatabaseWords = chksql(
@ -665,7 +683,6 @@ class Geocode
}
$aWordFrequencyScores[$aToken['word_id']] = $aToken['search_name_count'] + 1;
}
if (CONST_Debug) var_Dump($aPhrases, $aValidTokens);
// US ZIP+4 codes - if there is no token, merge in the 5-digit ZIP code
foreach ($aTokens as $sToken) {
@ -690,9 +707,11 @@ class Geocode
$aValidTokens[' '.$sToken] = array(array('class' => 'place', 'type' => 'house', 'word_token' => ' '.$sToken));
}
}
Debug::printGroupTable('Valid Tokens', $aValidTokens);
// Any words that have failed completely?
// TODO: suggestions
Debug::newSection('Search candidates');
$aGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $aValidTokens, $bStructuredPhrases);
@ -806,7 +825,7 @@ class Geocode
$aFilteredIDs = array();
if ($aFilterSql) {
$sSQL = join(' UNION ', $aFilterSql);
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
$aFilteredIDs = chksql($this->oDB->getCol($sSQL));
}
@ -835,7 +854,7 @@ class Geocode
$oLookup = $oReverse->lookupPoint($oCtx->sqlNear, false);
if (CONST_Debug) var_dump('Reverse search', $aLookup);
Debug::printVar('Reverse search', $oLookup);
if ($oLookup) {
$aResults = array($oLookup->iId => $oLookup);
@ -870,10 +889,7 @@ class Geocode
if (!preg_match('/[\pL\pN]/', $sWord)) unset($aRecheckWords[$i]);
}
if (CONST_Debug) {
echo '<i>Recheck words:<\i>';
var_dump($aRecheckWords);
}
Debug::printVar('Recheck words', $aRecheckWords);
foreach ($aSearchResults as $iIdx => $aResult) {
// Default
@ -952,18 +968,16 @@ class Geocode
$aResult['foundorder'] += 0.01;
}
}
if (CONST_Debug) var_dump($aResult);
$aSearchResults[$iIdx] = $aResult;
}
uasort($aSearchResults, 'byImportance');
Debug::printVar('Pre-filter results', $aSearchResults);
$aOSMIDDone = array();
$aClassTypeNameDone = array();
$aToFilter = $aSearchResults;
$aSearchResults = array();
if (CONST_Debug) var_dump($aToFilter);
$bFirst = true;
foreach ($aToFilter as $aResult) {
$this->aExcludePlaceIDs[$aResult['place_id']] = $aResult['place_id'];
@ -985,7 +999,30 @@ class Geocode
if (count($aSearchResults) >= $this->iFinalLimit) break;
}
if (CONST_Debug) var_dump($aSearchResults);
Debug::printVar('Post-filter results', $aSearchResults);
return $aSearchResults;
} // end lookup()
public function debugInfo()
{
return array(
'Query' => $this->sQuery,
'Structured query' => $this->aStructuredQuery,
'Name keys' => Debug::fmtArrayVals($this->aLangPrefOrder),
'Include address' => $this->bIncludeAddressDetails,
'Excluded place IDs' => Debug::fmtArrayVals($this->aExcludePlaceIDs),
'Try reversed query'=> $this->bReverseInPlan,
'Limit (for searches)' => $this->iLimit,
'Limit (for results)'=> $this->iFinalLimit,
'Country codes' => Debug::fmtArrayVals($this->aCountryCodes),
'Bounded search' => $this->bBoundedSearch,
'Viewbox' => Debug::fmtArrayVals($this->aViewBox),
'Route points' => Debug::fmtArrayVals($this->aRoutePoints),
'Route width' => $this->aRouteWidth,
'Max rank' => $this->iMaxRank,
'Min address rank' => $this->iMinAddressRank,
'Max address rank' => $this->iMaxAddressRank,
'Address rank list' => Debug::fmtArrayVals($this->aAddressRankList)
);
}
} // end class

View File

@ -113,4 +113,14 @@ class Phrase
return $aResult;
}
public function debugInfo()
{
return array(
'Type' => $this->sPhraseType,
'Phrase' => $this->sPhrase,
'Words' => $this->aWords,
'WordSets' => $this->aWordSets
);
}
}

View File

@ -169,14 +169,16 @@ class PlaceLookup
public function lookup($aResults, $iMinRank = 0, $iMaxRank = 30)
{
Debug::newFunction('Place lookup');
if (empty($aResults)) {
return array();
}
$aSubSelects = array();
$sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_PLACEX);
if (CONST_Debug) var_dump('PLACEX', $sPlaceIDs);
if ($sPlaceIDs) {
Debug::printVar('Ids from placex', $sPlaceIDs);
$sSQL = 'SELECT ';
$sSQL .= ' osm_type,';
$sSQL .= ' osm_id,';
@ -246,6 +248,7 @@ class PlaceLookup
// postcode table
$sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_POSTCODE);
if ($sPlaceIDs) {
Debug::printVar('Ids from location_postcode', $sPlaceIDs);
$sSQL = 'SELECT';
$sSQL .= " 'P' as osm_type,";
$sSQL .= ' (SELECT osm_id from placex p WHERE p.place_id = lp.parent_place_id) as osm_id,';
@ -276,6 +279,7 @@ class PlaceLookup
if (CONST_Use_US_Tiger_Data) {
$sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_TIGER);
if ($sPlaceIDs) {
Debug::printVar('Ids from Tiger table', $sPlaceIDs);
$sHousenumbers = Result::sqlHouseNumberTable($aResults, Result::TABLE_TIGER);
// Tiger search only if a housenumber was searched and if it was found
// (realized through a join)
@ -321,6 +325,7 @@ class PlaceLookup
// osmline - interpolated housenumbers
$sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_OSMLINE);
if ($sPlaceIDs) {
Debug::printVar('Ids from interpolation', $sPlaceIDs);
$sHousenumbers = Result::sqlHouseNumberTable($aResults, Result::TABLE_OSMLINE);
// interpolation line search only if a housenumber was searched
// (realized through a join)
@ -406,16 +411,13 @@ class PlaceLookup
}
}
if (CONST_Debug) var_dump($aSubSelects);
if (empty($aSubSelects)) {
return array();
}
$aPlaces = chksql(
$this->oDB->getAll(join(' UNION ', $aSubSelects)),
'Could not lookup place'
);
$sSQL = join(' UNION ', $aSubSelects);
Debug::printSQL($sSQL);
$aPlaces = chksql($this->oDB->getAll($sSQL), 'Could not lookup place');
$aClassType = getClassTypes();
foreach ($aPlaces as &$aPlace) {
@ -457,7 +459,7 @@ class PlaceLookup
$aPlace['addresstype'] = $sAddressType;
}
if (CONST_Debug) var_dump($aPlaces);
Debug::printVar('Places', $aPlaces);
return $aPlaces;
}

View File

@ -27,6 +27,17 @@ class Result
/// Subranking within the results (the higher the worse).
public $iResultRank = 0;
public function debugInfo()
{
return array(
'Table' => $this->iTable,
'ID' => $this->iId,
'House number' => $this->iHouseNumber,
'Exact Matches' => $this->iExactMatches,
'Result rank' => $this->iResultRank
);
}
public function __construct($sId, $iTable = Result::TABLE_PLACEX)
{

View File

@ -267,4 +267,18 @@ class SearchContext
return '';
}
public function debugInfo()
{
return array(
'Near radius' => $this->fNearRadius,
'Near point (SQL)' => $this->sqlNear,
'Bounded viewbox' => $this->bViewboxBounded,
'Viewbox (SQL, small)' => $this->sqlViewboxSmall,
'Viewbox (SQL, large)' => $this->sqlViewboxLarge,
'Viewbox (SQL, centre)' => $this->sqlViewboxCentre,
'Countries (SQL)' => $this->sqlCountryList,
'Excluded IDs (SQL)' => $this->sqlExcludeList
);
}
}

View File

@ -43,7 +43,6 @@ class SearchDescription
/// Index of phrase currently processed.
private $iNamePhrase = -1;
/**
* Create an empty search description.
*
@ -450,10 +449,7 @@ class SearchDescription
}
}
if (CONST_Debug) {
echo '<br><b>Place IDs:</b> ';
var_dump(array_keys($aResults));
}
Debug::printDebugTable('Place IDs', $aResults);
if (!empty($aResults) && $this->sPostcode) {
$sPlaceIds = Result::joinIdsByTable($aResults, Result::TABLE_PLACEX);
@ -461,7 +457,7 @@ class SearchDescription
$sSQL = 'SELECT place_id FROM placex';
$sSQL .= ' WHERE place_id in ('.$sPlaceIds.')';
$sSQL .= " AND postcode = '".$this->sPostcode."'";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
$aFilteredPlaceIDs = chksql($oDB->getCol($sSQL));
if ($aFilteredPlaceIDs) {
$aNewResults = array();
@ -469,10 +465,7 @@ class SearchDescription
$aNewResults[$iPlaceId] = $aResults[$iPlaceId];
}
$aResults = $aNewResults;
if (CONST_Debug) {
echo '<br><b>Place IDs after postcode filtering:</b> ';
var_dump(array_keys($aResults));
}
Debug::printVar('Place IDs after postcode filtering', $aResults);
}
}
}
@ -491,7 +484,7 @@ class SearchDescription
}
$sSQL .= ' ORDER BY st_area(geometry) DESC LIMIT 1';
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
$aResults = array();
foreach (chksql($oDB->getCol($sSQL)) as $iPlaceId) {
@ -532,7 +525,7 @@ class SearchDescription
$sSQL .= ' ORDER BY '.$this->oContext->distanceSQL('ct.centroid').' ASC';
}
$sSQL .= " limit $iLimit";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
$aDBResults = chksql($oDB->getCol($sSQL));
}
@ -546,7 +539,7 @@ class SearchDescription
}
$sSQL .= ' ORDER BY '.$this->oContext->distanceSQL('centroid').' ASC';
$sSQL .= " LIMIT $iLimit";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
$aDBResults = chksql($oDB->getCol($sSQL));
}
@ -576,7 +569,7 @@ class SearchDescription
$sSQL .= $this->oContext->excludeSQL(' AND p.place_id');
$sSQL .= " LIMIT $iLimit";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
$aResults = array();
foreach (chksql($oDB->getCol($sSQL)) as $iPlaceId) {
@ -707,7 +700,7 @@ class SearchDescription
$sSQL .= ' ORDER BY '.join(', ', $aOrder);
$sSQL .= ' LIMIT '.$iLimit;
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
$aDBResults = chksql(
$oDB->getAll($sSQL),
@ -740,7 +733,7 @@ class SearchDescription
$sSQL .= $this->oContext->excludeSQL(' AND place_id');
$sSQL .= " LIMIT $iLimit";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
// XXX should inherit the exactMatches from its parent
foreach (chksql($oDB->getCol($sSQL)) as $iPlaceId) {
@ -768,7 +761,7 @@ class SearchDescription
$sSQL .= $this->oContext->excludeSQL(' AND place_id');
$sSQL .= " limit $iLimit";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
foreach (chksql($oDB->getCol($sSQL)) as $iPlaceId) {
$oResult = new Result($iPlaceId, Result::TABLE_OSMLINE);
@ -785,7 +778,7 @@ class SearchDescription
$sSQL .= $this->oContext->excludeSQL(' AND place_id');
$sSQL .= " limit $iLimit";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
foreach (chksql($oDB->getCol($sSQL)) as $iPlaceId) {
$aResults[$iPlaceId] = new Result($iPlaceId, Result::TABLE_AUX);
@ -807,7 +800,7 @@ class SearchDescription
$sSQL .= $this->oContext->excludeSQL(' AND place_id');
$sSQL .= " limit $iLimit";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
foreach (chksql($oDB->getCol($sSQL)) as $iPlaceId) {
$oResult = new Result($iPlaceId, Result::TABLE_TIGER);
@ -841,7 +834,7 @@ class SearchDescription
$sSQL .= ' ORDER BY rank_search ASC ';
$sSQL .= " LIMIT $iLimit";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
foreach (chksql($oDB->getCol($sSQL)) as $iPlaceId) {
$aResults[$iPlaceId] = new Result($iPlaceId);
@ -855,7 +848,7 @@ class SearchDescription
$bCacheTable = (bool) chksql($oDB->getOne($sSQL));
$sSQL = "SELECT min(rank_search) FROM placex WHERE place_id in ($sPlaceIDs)";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
$iMaxRank = (int)chksql($oDB->getOne($sSQL));
// For state / country level searches the normal radius search doesn't work very well
@ -868,7 +861,7 @@ class SearchDescription
$sSQL .= " AND ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon')";
$sSQL .= ' ORDER BY rank_search ASC ';
$sSQL .= ' LIMIT 1';
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
$sPlaceGeom = chksql($oDB->getOne($sSQL));
}
@ -878,7 +871,7 @@ class SearchDescription
$iMaxRank += 5;
$sSQL = 'SELECT place_id FROM placex';
$sSQL .= " WHERE place_id in ($sPlaceIDs) and rank_search < $iMaxRank";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
$aPlaceIDs = chksql($oDB->getCol($sSQL));
$sPlaceIDs = join(',', $aPlaceIDs);
}
@ -923,7 +916,7 @@ class SearchDescription
}
$sSQL .= " limit $iLimit";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
foreach (chksql($oDB->getCol($sSQL)) as $iPlaceId) {
$aResults[$iPlaceId] = new Result($iPlaceId);
@ -955,7 +948,7 @@ class SearchDescription
}
$sSQL .= " limit $iLimit";
if (CONST_Debug) var_dump($sSQL);
Debug::printSQL($sSQL);
foreach (chksql($oDB->getCol($sSQL)) as $iPlaceId) {
$aResults[$iPlaceId] = new Result($iPlaceId);
@ -1000,6 +993,24 @@ class SearchDescription
//////////// Debugging functions
public function debugInfo()
{
return array(
'Search rank' => $this->iSearchRank,
'Country code' => $this->sCountryCode,
'Name terms' => $this->aName,
'Name terms (stop words)' => $this->aNameNonSearch,
'Address terms' => $this->aAddress,
'Address terms (stop words)' => $this->aAddressNonSearch,
'Address terms (full words)' => $this->aFullNameAddress,
'Special search' => $this->iOperator,
'Class' => $this->sClass,
'Type' => $this->sType,
'House number' => $this->sHouseNumber,
'Postcode' => $this->sPostcode
);
}
public function dumpAsHtmlTableRow(&$aWordIDs)
{
$kf = function ($k) use (&$aWordIDs) {

View File

@ -2,6 +2,7 @@
require_once('init.php');
require_once('cmd.php');
require_once('DebugNone.php');
// handle http proxy when using file_get_contents
if (CONST_HTTP_Proxy) {

View File

@ -2,6 +2,7 @@
require_once('init.php');
require_once('ParameterParser.php');
require_once(CONST_Debug ? 'DebugHtml.php' : 'DebugNone.php');
/***************************************************************************
*