mirror of
https://github.com/osm-search/Nominatim.git
synced 2024-11-23 05:35:13 +03:00
index on geometry of interpolation lines, and more improvements.
This commit is contained in:
Markus Gail
parent
a10f537131
commit
db719d489f
@ -466,10 +466,10 @@
|
||||
$sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
|
||||
if (!$this->bDeDupe) $sSQL .= ", place_id ";
|
||||
}
|
||||
// osmline, osm_type is 'I' for Interpolation Line
|
||||
// osmline
|
||||
// interpolation line search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
|
||||
$sSQL .= " union ";
|
||||
$sSQL .= "select 'I' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, calculated_country_code as country_code, ";
|
||||
$sSQL .= "select 'W' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, calculated_country_code as country_code, ";
|
||||
$sSQL .= "get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress, ";
|
||||
$sSQL .= "null as placename, ";
|
||||
$sSQL .= "null as ref, ";
|
||||
@ -477,7 +477,8 @@
|
||||
if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
|
||||
$sSQL .= " avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
|
||||
$sSQL .= $sImportanceSQL."-0.1 as importance, "; // slightly smaller than the importance for normal houses with rank 30, which is 0
|
||||
$sSQL .= " (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
|
||||
$sSQL .= " (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p";
|
||||
$sSQL .= " where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance,";
|
||||
$sSQL .= " null as extra_place ";
|
||||
$sSQL .= " from (select place_id, calculated_country_code ";
|
||||
//interpolate the housenumbers here
|
||||
@ -1454,7 +1455,7 @@
|
||||
$aRoadPlaceIDs = $aPlaceIDs;
|
||||
$sPlaceIDs = join(',',$aPlaceIDs);
|
||||
|
||||
// Now they are indexed look for a house attached to a street we found
|
||||
// Now they are indexed, look for a house attached to a street we found
|
||||
$sHouseNumberRegex = '\\\\m'.$aSearch['sHouseNumber'].'\\\\M';
|
||||
$sSQL = "select place_id from placex where parent_place_id in (".$sPlaceIDs.") and transliteration(housenumber) ~* E'".$sHouseNumberRegex."'";
|
||||
if (sizeof($this->aExcludePlaceIDs))
|
||||
|
||||
@ -142,9 +142,9 @@
|
||||
}
|
||||
else if ($this->sType == 'interpolation')
|
||||
{
|
||||
$sSQL = "select place_id, partition, 'I' as osm_type, osm_id, 'place' as class, 'house' as type, null admin_level, housenumber, null as street, null as isin, postcode,";
|
||||
$sSQL = "select place_id, partition, 'W' as osm_type, osm_id, 'place' as class, 'house' as type, null admin_level, housenumber, null as street, null as isin, postcode,";
|
||||
$sSQL .= " calculated_country_code as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
|
||||
$sSQL .= " coalesce(null,0.75-(30::float/40)) as importance, null as indexed_status, null as indexed_date, null as wikipedia, calculated_country_code, ";
|
||||
$sSQL .= " (0.75-(30::float/40)) as importance, null as indexed_status, null as indexed_date, null as wikipedia, calculated_country_code, ";
|
||||
$sSQL .= " get_address_by_language(place_id, housenumber, $sLanguagePrefArraySQL) as langaddress,";
|
||||
$sSQL .= " null as placename,";
|
||||
$sSQL .= " null as ref,";
|
||||
@ -187,7 +187,7 @@
|
||||
|
||||
if ($this->bAddressDetails)
|
||||
{
|
||||
if(CONST_Use_US_Tiger_Data && $this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed
|
||||
if(CONST_Use_US_Tiger_Data && $this->sType == 'tiger' || $this->sType == 'interpolation') // to get addressdetails for tiger data, the housenumber is needed
|
||||
$aAddress = $this->getAddressNames($aPlace['housenumber']);
|
||||
else
|
||||
$aAddress = $this->getAddressNames();
|
||||
|
||||
@ -158,43 +158,15 @@
|
||||
$bIsInUnitedStates = ($aPlace['calculated_country_code'] == 'us');
|
||||
}
|
||||
// if a street or house was found, look in interpolation lines table
|
||||
if ($iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 || $aPlace['rank_search'] == 30))
|
||||
if ($iMaxRank_orig >= 28 && $aPlace && $aPlace['rank_search'] >= 26)
|
||||
{
|
||||
$fSearchDiam = 0.001;
|
||||
if ($aPlace['rank_search'] == 30)
|
||||
{
|
||||
// if a house was found, the closest road needs to be searched, to use its place id as parent_place_id for the interpolation line search
|
||||
// because a road can be closer to the point than the house from above
|
||||
$iRoadID = null;
|
||||
while(!$iRoadID && $fSearchDiam < $fMaxAreaDistance)
|
||||
{
|
||||
$fSearchDiam = $fSearchDiam * 2;
|
||||
$sSQL = 'select place_id ';
|
||||
$sSQL .= ' FROM placex';
|
||||
$sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')';
|
||||
$sSQL .= ' and (rank_search = 26 or rank_search = 27)';
|
||||
$sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\',\'man_made\')';
|
||||
$sSQL .= ' and indexed_status = 0 ';
|
||||
$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1';
|
||||
$aPlaceRoad = $this->oDB->getRow($sSQL);
|
||||
if (PEAR::IsError($aPlace))
|
||||
{
|
||||
failInternalError("Could not determine closest place.", $sSQL, $aPlace);
|
||||
}
|
||||
$iRoadID = $aPlaceRoad['place_id'];
|
||||
$iTempPlaceID = $iRoadID;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// if a street was found, we can take its place_id as parent_place_id
|
||||
$iTempPlaceID = $iPlaceID;
|
||||
}
|
||||
// if a house was found, search the interpolation line that is at least as close as the house
|
||||
$sSQL = 'SELECT place_id, parent_place_id, 30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction';
|
||||
//if (CONST_Debug) { $sSQL .= ', housenumber, ST_distance('.$sPointSQL.', centroid) as distance, st_y(centroid) as lat, st_x(centroid) as lon'; }
|
||||
$sSQL .= ' FROM location_property_osmline WHERE parent_place_id = '.$iTempPlaceID;
|
||||
$sSQL .= ' AND ST_DWithin('.$sPointSQL.', linegeo, '.$fSearchDiam.') AND indexed_status = 0';
|
||||
$sSQL .= ' FROM location_property_osmline';
|
||||
$sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', linegeo, '.$fSearchDiam.')';
|
||||
$sSQL .= ' and indexed_status = 0 ';
|
||||
$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', linegeo) ASC limit 1';
|
||||
|
||||
if (CONST_Debug)
|
||||
{
|
||||
$sSQL = preg_replace('/limit 1/', 'limit 100', $sSQL);
|
||||
@ -211,7 +183,6 @@
|
||||
{
|
||||
failInternalError("Could not determine closest housenumber on an osm interpolation line.", $sSQL, $aPlaceLine);
|
||||
}
|
||||
$iInterpolationLinePlaceID = $aPlaceLine['place_id'];
|
||||
if ($aPlaceLine)
|
||||
{
|
||||
if (CONST_Debug) var_dump('found housenumber in interpolation lines table', $aPlaceLine);
|
||||
@ -229,7 +200,7 @@
|
||||
$fDistancePlacex = $aDistancePlacex['distance'];
|
||||
// distance between point and interpolated house (fraction on interpolation line)
|
||||
$sSQL = 'SELECT ST_distance('.$sPointSQL.', ST_LineInterpolatePoint(linegeo, '.$aPlaceLine['fraction'].')) as distance';
|
||||
$sSQL .= ' FROM location_property_osmline WHERE place_id = '.$iInterpolationLinePlaceID;
|
||||
$sSQL .= ' FROM location_property_osmline WHERE place_id = '.$aPlaceLine['place_id'];
|
||||
$aDistanceInterpolation = $this->oDB->getRow($sSQL);
|
||||
if (PEAR::IsError($aDistanceInterpolation))
|
||||
{
|
||||
@ -241,7 +212,7 @@
|
||||
// interpolation is closer to point than placex house
|
||||
$bPlaceIsLine = true;
|
||||
$aPlace = $aPlaceLine;
|
||||
$iPlaceID = $iInterpolationLinePlaceID;
|
||||
$iPlaceID = $aPlaceLine['place_id'];
|
||||
$iParentPlaceID = $aPlaceLine['parent_place_id']; // the street
|
||||
$fFraction = $aPlaceLine['fraction'];
|
||||
}
|
||||
|
||||
@ -21,34 +21,243 @@
|
||||
|
||||
extern int verbose;
|
||||
|
||||
void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile)
|
||||
void run_indexing(int rank, int interpolation, PGconn *conn, int num_threads,
|
||||
struct index_thread_data * thread_data, const char *structuredoutputfile)
|
||||
{
|
||||
struct index_thread_data * thread_data;
|
||||
pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
int tuples, count, sleepcount;
|
||||
|
||||
pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
time_t rankStartTime;
|
||||
int rankTotalTuples;
|
||||
int rankCountTuples;
|
||||
float rankPerSecond;
|
||||
|
||||
PGconn *conn;
|
||||
PGresult * res;
|
||||
|
||||
PGresult * resSectors;
|
||||
PGresult * resPlaces;
|
||||
PGresult * resNULL;
|
||||
|
||||
int rank;
|
||||
|
||||
int i;
|
||||
int iSector;
|
||||
int iResult;
|
||||
|
||||
|
||||
const char *paramValues[2];
|
||||
int paramLengths[2];
|
||||
int paramFormats[2];
|
||||
uint32_t paramRank;
|
||||
uint32_t paramSector;
|
||||
uint32_t sector;
|
||||
|
||||
xmlTextWriterPtr writer;
|
||||
pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
// Create the output file
|
||||
writer = NULL;
|
||||
if (structuredoutputfile)
|
||||
{
|
||||
writer = nominatim_exportXMLStart(structuredoutputfile);
|
||||
}
|
||||
|
||||
if (interpolation)
|
||||
{
|
||||
fprintf(stderr, "Starting interpolation lines (location_property_osmline)\n");
|
||||
}
|
||||
else
|
||||
{
|
||||
fprintf(stderr, "Starting rank %d\n", rank);
|
||||
}
|
||||
|
||||
rankCountTuples = 0;
|
||||
rankPerSecond = 0;
|
||||
|
||||
paramRank = PGint32(rank);
|
||||
paramValues[0] = (char *)¶mRank;
|
||||
paramLengths[0] = sizeof(paramRank);
|
||||
paramFormats[0] = 1;
|
||||
|
||||
if (interpolation)
|
||||
{
|
||||
resSectors = PQexecPrepared(conn, "index_sectors_osmline", 0, NULL, 0, NULL, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 0) != PG_OID_INT4)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 1) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
rankTotalTuples = 0;
|
||||
for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
|
||||
{
|
||||
rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)));
|
||||
}
|
||||
|
||||
rankStartTime = time(0);
|
||||
for (iSector = 0; iSector <= PQntuples(resSectors); iSector++)
|
||||
{
|
||||
if (iSector > 0)
|
||||
{
|
||||
resPlaces = PQgetResult(conn);
|
||||
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resPlaces, 0) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Place_id value has unexpected type\n");
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
resNULL = PQgetResult(conn);
|
||||
if (resNULL != NULL)
|
||||
{
|
||||
fprintf(stderr, "Unexpected non-null response\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (iSector < PQntuples(resSectors))
|
||||
{
|
||||
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
|
||||
// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
|
||||
|
||||
// Get all the place_id's for this sector
|
||||
paramRank = PGint32(rank);
|
||||
paramValues[0] = (char *)¶mRank;
|
||||
paramLengths[0] = sizeof(paramRank);
|
||||
paramFormats[0] = 1;
|
||||
paramSector = PGint32(sector);
|
||||
paramValues[1] = (char *)¶mSector;
|
||||
paramLengths[1] = sizeof(paramSector);
|
||||
paramFormats[1] = 1;
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*1000)
|
||||
{
|
||||
// no sectors
|
||||
if (interpolation)
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_nosector_places_osmline", 0, NULL, 0, NULL, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (interpolation)
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_sector_places_osmline", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
}
|
||||
if (!iResult)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
if (iSector > 0)
|
||||
{
|
||||
count = 0;
|
||||
rankPerSecond = 0;
|
||||
tuples = PQntuples(resPlaces);
|
||||
|
||||
if (tuples > 0)
|
||||
{
|
||||
// Spawn threads
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
thread_data[i].res = resPlaces;
|
||||
thread_data[i].tuples = tuples;
|
||||
thread_data[i].count = &count;
|
||||
thread_data[i].count_mutex = &count_mutex;
|
||||
thread_data[i].writer = writer;
|
||||
thread_data[i].writer_mutex = &writer_mutex;
|
||||
if (interpolation)
|
||||
{
|
||||
thread_data[i].table = 0; // use interpolations table
|
||||
}
|
||||
else
|
||||
{
|
||||
thread_data[i].table = 1; // use placex table
|
||||
}
|
||||
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
|
||||
}
|
||||
|
||||
// Monitor threads to give user feedback
|
||||
sleepcount = 0;
|
||||
while (count < tuples)
|
||||
{
|
||||
usleep(1000);
|
||||
|
||||
// Aim for one update per second
|
||||
if (sleepcount++ > 500)
|
||||
{
|
||||
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
|
||||
sleepcount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for everything to finish
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
pthread_join(thread_data[i].thread, NULL);
|
||||
}
|
||||
|
||||
rankCountTuples += tuples;
|
||||
}
|
||||
|
||||
// Finished sector
|
||||
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
|
||||
|
||||
PQclear(resPlaces);
|
||||
}
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
|
||||
{
|
||||
iSector = PQntuples(resSectors) - 1;
|
||||
}
|
||||
}
|
||||
// Finished rank
|
||||
fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED\n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
|
||||
|
||||
PQclear(resSectors);
|
||||
|
||||
|
||||
}
|
||||
|
||||
void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile)
|
||||
{
|
||||
struct index_thread_data * thread_data;
|
||||
|
||||
PGconn *conn;
|
||||
PGresult * res;
|
||||
|
||||
int rank;
|
||||
|
||||
int i;
|
||||
|
||||
xmlTextWriterPtr writer;
|
||||
pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
@ -182,324 +391,22 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
|
||||
nominatim_exportCreatePreparedQueries(thread_data[i].conn);
|
||||
}
|
||||
|
||||
// Create the output file
|
||||
writer = NULL;
|
||||
if (structuredoutputfile)
|
||||
{
|
||||
writer = nominatim_exportXMLStart(structuredoutputfile);
|
||||
}
|
||||
|
||||
fprintf(stderr, "Starting indexing rank (%i to %i) using %i threads\n", rank_min, rank_max, num_threads);
|
||||
|
||||
// first for the placex table
|
||||
for (rank = rank_min; rank <= rank_max; rank++)
|
||||
{
|
||||
// OSMLINE: do reindexing (=> reparenting) for interpolation lines at rank 30, but before all other objects of rank 30
|
||||
// reason: houses (rank 30) depend on the updated interpolation line, when reparenting (see placex_update in functions.sql)
|
||||
if (rank == 30)
|
||||
{
|
||||
fprintf(stderr, "Starting indexing interpolation lines (location_property_osmline)\n");
|
||||
rankCountTuples = 0;
|
||||
rankTotalTuples = 0;
|
||||
resSectors = PQexecPrepared(conn, "index_sectors_osmline", 0, NULL, 0, NULL, 1);
|
||||
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sectors_osmline: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 0) != PG_OID_INT4)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 1) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
rankStartTime = time(0);
|
||||
for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
|
||||
{
|
||||
rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)));
|
||||
}
|
||||
// do it only if tuples with indexed_status > 0 were found in osmline
|
||||
int nTuples = PQntuples(resSectors);
|
||||
if (nTuples > 0)
|
||||
{
|
||||
for (iSector = 0; iSector <= nTuples; iSector++)
|
||||
{
|
||||
if (iSector > 0)
|
||||
{
|
||||
resPlaces = PQgetResult(conn);
|
||||
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places: SELECT failed: %s\n", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resPlaces, 0) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Place_id value has unexpected type\n");
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
resNULL = PQgetResult(conn);
|
||||
if (resNULL != NULL)
|
||||
{
|
||||
fprintf(stderr, "Unexpected non-null response\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (iSector < nTuples)
|
||||
{
|
||||
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
|
||||
// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
|
||||
|
||||
// Get all the place_id's for this sector
|
||||
paramSector = PGint32(sector);
|
||||
paramValues[0] = (char *)¶mSector;
|
||||
paramLengths[0] = sizeof(paramSector);
|
||||
paramFormats[0] = 1;
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*1000)
|
||||
{
|
||||
// no sectors
|
||||
iResult = PQsendQueryPrepared(conn, "index_nosector_places_osmline", 0, NULL, 0, NULL, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_sector_places_osmline", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
if (!iResult)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places_osmline: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
if (iSector > 0)
|
||||
{
|
||||
count = 0;
|
||||
rankPerSecond = 0;
|
||||
tuples = PQntuples(resPlaces);
|
||||
|
||||
if (tuples > 0)
|
||||
{
|
||||
// Spawn threads
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
thread_data[i].res = resPlaces;
|
||||
thread_data[i].tuples = tuples;
|
||||
thread_data[i].count = &count;
|
||||
thread_data[i].count_mutex = &count_mutex;
|
||||
thread_data[i].writer = writer;
|
||||
thread_data[i].writer_mutex = &writer_mutex;
|
||||
thread_data[i].table = 0; // use osmline table
|
||||
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
|
||||
}
|
||||
// Monitor threads to give user feedback
|
||||
sleepcount = 0;
|
||||
while (count < tuples)
|
||||
{
|
||||
usleep(1000);
|
||||
|
||||
// Aim for one update per second
|
||||
if (sleepcount++ > 500)
|
||||
{
|
||||
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - Interpolation Lines ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - (rankCountTuples + count)))/(float)rankPerSecond);
|
||||
sleepcount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for everything to finish
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
pthread_join(thread_data[i].thread, NULL);
|
||||
}
|
||||
rankCountTuples += tuples;
|
||||
}
|
||||
// Finished sector
|
||||
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
|
||||
PQclear(resPlaces);
|
||||
}
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < nTuples)
|
||||
{
|
||||
iSector = nTuples - 1;
|
||||
}
|
||||
}
|
||||
PQclear(resSectors);
|
||||
}
|
||||
// Finished rank
|
||||
fprintf(stderr, "\r Done %i tuples in %i seconds- FINISHED\n", rankCountTuples,(int)(difftime(time(0), rankStartTime)));
|
||||
if (writer)
|
||||
{
|
||||
nominatim_exportXMLEnd(writer);
|
||||
}
|
||||
run_indexing(rank, 1, conn, num_threads, thread_data, structuredoutputfile);
|
||||
}
|
||||
fprintf(stderr, "Starting rank %d\n", rank);
|
||||
rankCountTuples = 0;
|
||||
rankPerSecond = 0;
|
||||
|
||||
paramRank = PGint32(rank);
|
||||
paramValues[0] = (char *)¶mRank;
|
||||
paramLengths[0] = sizeof(paramRank);
|
||||
paramFormats[0] = 1;
|
||||
// if (rank < 16)
|
||||
// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
// else
|
||||
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
|
||||
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 0) != PG_OID_INT4)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 1) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
rankTotalTuples = 0;
|
||||
for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
|
||||
{
|
||||
rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)));
|
||||
}
|
||||
|
||||
rankStartTime = time(0);
|
||||
|
||||
for (iSector = 0; iSector <= PQntuples(resSectors); iSector++)
|
||||
{
|
||||
if (iSector > 0)
|
||||
{
|
||||
resPlaces = PQgetResult(conn);
|
||||
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resPlaces, 0) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Place_id value has unexpected type\n");
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
resNULL = PQgetResult(conn);
|
||||
if (resNULL != NULL)
|
||||
{
|
||||
fprintf(stderr, "Unexpected non-null response\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (iSector < PQntuples(resSectors))
|
||||
{
|
||||
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
|
||||
// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
|
||||
|
||||
// Get all the place_id's for this sector
|
||||
paramRank = PGint32(rank);
|
||||
paramValues[0] = (char *)¶mRank;
|
||||
paramLengths[0] = sizeof(paramRank);
|
||||
paramFormats[0] = 1;
|
||||
paramSector = PGint32(sector);
|
||||
paramValues[1] = (char *)¶mSector;
|
||||
paramLengths[1] = sizeof(paramSector);
|
||||
paramFormats[1] = 1;
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*1000)
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
if (!iResult)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (iSector > 0)
|
||||
{
|
||||
count = 0;
|
||||
rankPerSecond = 0;
|
||||
tuples = PQntuples(resPlaces);
|
||||
|
||||
if (tuples > 0)
|
||||
{
|
||||
// Spawn threads
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
thread_data[i].res = resPlaces;
|
||||
thread_data[i].tuples = tuples;
|
||||
thread_data[i].count = &count;
|
||||
thread_data[i].count_mutex = &count_mutex;
|
||||
thread_data[i].writer = writer;
|
||||
thread_data[i].writer_mutex = &writer_mutex;
|
||||
thread_data[i].table = 1; // use placex table
|
||||
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
|
||||
}
|
||||
|
||||
// Monitor threads to give user feedback
|
||||
sleepcount = 0;
|
||||
while (count < tuples)
|
||||
{
|
||||
usleep(1000);
|
||||
|
||||
// Aim for one update per second
|
||||
if (sleepcount++ > 500)
|
||||
{
|
||||
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
|
||||
sleepcount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for everything to finish
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
pthread_join(thread_data[i].thread, NULL);
|
||||
}
|
||||
|
||||
rankCountTuples += tuples;
|
||||
}
|
||||
|
||||
// Finished sector
|
||||
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
|
||||
|
||||
PQclear(resPlaces);
|
||||
}
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
|
||||
{
|
||||
iSector = PQntuples(resSectors) - 1;
|
||||
}
|
||||
}
|
||||
// Finished rank
|
||||
fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
|
||||
|
||||
PQclear(resSectors);
|
||||
run_indexing(rank, 0, conn, num_threads, thread_data, structuredoutputfile);
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (rank_max == 30)
|
||||
if (rank == 30)
|
||||
{
|
||||
// Close all connections
|
||||
for (i = 0; i < num_threads; i++)
|
||||
|
||||
@ -1090,13 +1090,12 @@ BEGIN
|
||||
END IF;
|
||||
|
||||
IF OLD.indexed_status = 2 and NEW.indexed_status=0 THEN
|
||||
-- do the reparenting: (finally here, so that ALL places in placex, that are needed for reparenting, are up to date)
|
||||
-- do the reparenting: (finally here, because ALL places in placex, that are needed for reparenting, need to be up to date)
|
||||
-- (the osm interpolationline in location_property_osmline was marked for reparenting in placex_insert/placex_delete with index_status = 2
|
||||
-- => index.c: sets index_status back to 0
|
||||
-- => triggers this function)
|
||||
place_centroid := ST_PointOnSurface(NEW.linegeo);
|
||||
-- mark descendants for reparenting
|
||||
UPDATE placex SET indexed_status = 2 WHERE parent_place_id = OLD.place_id;
|
||||
-- marking descendants for reparenting is not needed, because there are actually no descendants for interpolation lines
|
||||
NEW.parent_place_id = get_interpolation_parent(NEW.osm_id, NEW.street, null, NEW.partition, place_centroid, NEW.linegeo); -- addr_place (3rd param) is not necessarily needed
|
||||
return NEW;
|
||||
END IF;
|
||||
@ -1156,14 +1155,6 @@ BEGIN
|
||||
END IF;
|
||||
|
||||
IF NEW.indexed_status != 0 OR OLD.indexed_status = 0 THEN
|
||||
|
||||
-- if a node(=>house), which is part of a interpolation line, changes (e.g. the street attribute) => mark this line for reparenting
|
||||
-- (already here, because interpolation lines are reindexed before nodes, so in the second call it would be too late)
|
||||
-- needed for test case features/db/import: Scenario: addr:street added to housenumbers
|
||||
IF NEW.osm_type='N' and NEW.class='place' and NEW.type='house' THEN
|
||||
-- Is this node part of an interpolation line? search for it in location_property_osmline and mark the interpolation line for reparenting
|
||||
update location_property_osmline p set indexed_status = 2 from planet_osm_ways w where p.linegeo && NEW.geometry and p.osm_id = w.id and NEW.osm_id = any(w.nodes);
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
|
||||
@ -1906,7 +1897,7 @@ BEGIN
|
||||
UPDATE placex set indexed_status = 100 where osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type;
|
||||
|
||||
-- interpolations are special
|
||||
IF OLD.class = 'place' and OLD.type = 'houses' THEN
|
||||
IF OLD.osm_type='W' and OLD.class = 'place' and OLD.type = 'houses' THEN
|
||||
UPDATE location_property_osmline set indexed_status = 100 where osm_id = OLD.osm_id; -- osm_id = wayid (=old.osm_id)
|
||||
END IF;
|
||||
|
||||
@ -1931,7 +1922,6 @@ BEGIN
|
||||
|
||||
--DEBUG: RAISE WARNING '-----------------------------------------------------------------------------------';
|
||||
--DEBUG: RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry);
|
||||
RAISE WARNING 'X3366 - place_isnert';
|
||||
-- filter wrong tupels
|
||||
IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN
|
||||
INSERT INTO import_polygon_error values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.country_code,
|
||||
@ -1960,12 +1950,6 @@ BEGIN
|
||||
|
||||
-- To paraphrase, if there isn't an existing item
|
||||
IF existingline.osm_id IS NULL THEN
|
||||
IF existing.osm_type IS NOT NULL THEN
|
||||
-- pathological case caused by the triggerless copy into place during initial import
|
||||
-- force delete even for large areas, it will be reinserted later
|
||||
UPDATE place set geometry = ST_SetSRID(ST_Point(0,0), 4326) where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
|
||||
DELETE from place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
|
||||
END IF;
|
||||
-- insert new line into location_property_osmline, use function insert_osmline
|
||||
i = insert_osmline(NEW.osm_id, NEW.housenumber, NEW.street, NEW.addr_place, NEW.postcode, NEW.country_code, NEW.geometry);
|
||||
RETURN NEW;
|
||||
@ -2002,7 +1986,6 @@ BEGIN
|
||||
|
||||
-- for interpolations invalidate all nodes on the line
|
||||
update placex p set indexed_status = 2 from planet_osm_ways w where w.id = NEW.osm_id and p.osm_type = 'N' and p.osm_id = any(w.nodes);
|
||||
RAISE WARNING 'X3399 - updated nodes of interpolation line';
|
||||
RETURN NULL;
|
||||
|
||||
ELSE -- insert to placex
|
||||
@ -2170,6 +2153,7 @@ BEGIN
|
||||
admin_level = NEW.admin_level,
|
||||
geometry = NEW.geometry
|
||||
where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
|
||||
|
||||
|
||||
IF NEW.class in ('place','boundary') AND NEW.type in ('postcode','postal_code') THEN
|
||||
IF NEW.postcode IS NULL THEN
|
||||
@ -2195,6 +2179,14 @@ BEGIN
|
||||
indexed_status = 2,
|
||||
geometry = NEW.geometry
|
||||
where place_id = existingplacex.place_id;
|
||||
|
||||
-- if a node(=>house), which is part of a interpolation line, changes (e.g. the street attribute) => mark this line for reparenting
|
||||
-- (already here, because interpolation lines are reindexed before nodes, so in the second call it would be too late)
|
||||
-- needed for test case features/db/import: Scenario: addr:street added to housenumbers
|
||||
IF NEW.osm_type='N' and NEW.class='place' and NEW.type='house' THEN
|
||||
-- Is this node part of an interpolation line? search for it in location_property_osmline and mark the interpolation line for reparenting
|
||||
update location_property_osmline p set indexed_status = 2 from planet_osm_ways w where p.linegeo && NEW.geometry and p.osm_id = w.id and NEW.osm_id = any(w.nodes);
|
||||
END IF;
|
||||
|
||||
END IF;
|
||||
|
||||
@ -2202,7 +2194,7 @@ BEGIN
|
||||
RETURN NULL;
|
||||
END IF;
|
||||
|
||||
END;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
|
||||
|
||||
@ -84,6 +84,7 @@ GRANT SELECT ON location_property_aux TO "{www-user}";
|
||||
CREATE TABLE location_property_tiger (linegeo GEOMETRY, place_id BIGINT, partition INTEGER, parent_place_id BIGINT, startnumber INTEGER, endnumber INTEGER, interpolationtype TEXT, postcode TEXT);
|
||||
GRANT SELECT ON location_property_tiger TO "{www-user}";
|
||||
|
||||
drop table if exists location_property_osmline;
|
||||
CREATE TABLE location_property_osmline (
|
||||
linegeo GEOMETRY,
|
||||
place_id BIGINT NOT NULL,
|
||||
@ -99,9 +100,11 @@ CREATE TABLE location_property_osmline (
|
||||
calculated_country_code VARCHAR(2),
|
||||
geometry_sector INTEGER,
|
||||
indexed_status INTEGER,
|
||||
indexed_date TIMESTAMP);
|
||||
CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline (place_id) {ts:search-index};
|
||||
CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline (parent_place_id) {ts:search-index};
|
||||
indexed_date TIMESTAMP){ts:search-data};
|
||||
CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {ts:search-index};
|
||||
CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {ts:search-index};
|
||||
CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {ts:address-index};
|
||||
CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {ts:search-index};
|
||||
GRANT SELECT ON location_property_osmline TO "{www-user}";
|
||||
|
||||
drop table IF EXISTS search_name;
|
||||
|
||||
@ -18,7 +18,10 @@ Feature: Reverse geocoding
|
||||
| xml | 4
|
||||
When looking up coordinates 53.9788769,13.0830313
|
||||
And results contain valid boundingboxes
|
||||
|
||||
|
||||
Scenario: Reverse geocoding for odd interpolated housenumber
|
||||
|
||||
Scenario: Reverse geocoding for even interpolated housenumber
|
||||
|
||||
@Tiger
|
||||
Scenario: TIGER house number
|
||||
|
||||
@ -37,7 +37,6 @@ Feature: Update of address interpolations
|
||||
| object | parent_place_id | startnumber | endnumber
|
||||
| W10 | W3 | 2 | 6
|
||||
|
||||
@Fail
|
||||
Scenario: addr:street added to housenumbers
|
||||
Given the scene parallel-road
|
||||
And the place nodes
|
||||
|
||||
Loading…
Reference in New Issue
Block a user