diff --git a/lib/Geocode.php b/lib/Geocode.php index 0fb333aa..61ac834b 100644 --- a/lib/Geocode.php +++ b/lib/Geocode.php @@ -466,10 +466,10 @@ $sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique if (!$this->bDeDupe) $sSQL .= ", place_id "; } - // osmline, osm_type is 'I' for Interpolation Line + // osmline // interpolation line search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join) $sSQL .= " union "; - $sSQL .= "select 'I' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, calculated_country_code as country_code, "; + $sSQL .= "select 'W' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, calculated_country_code as country_code, "; $sSQL .= "get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress, "; $sSQL .= "null as placename, "; $sSQL .= "null as ref, "; @@ -477,7 +477,8 @@ if ($this->bIncludeNameDetails) $sSQL .= "null as names, "; $sSQL .= " avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,"; $sSQL .= $sImportanceSQL."-0.1 as importance, "; // slightly smaller than the importance for normal houses with rank 30, which is 0 - $sSQL .= " (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; + $sSQL .= " (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p"; + $sSQL .= " where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance,"; $sSQL .= " null as extra_place "; $sSQL .= " from (select place_id, calculated_country_code "; //interpolate the housenumbers here @@ -1454,7 +1455,7 @@ $aRoadPlaceIDs = $aPlaceIDs; $sPlaceIDs = join(',',$aPlaceIDs); - // Now they are indexed look for a house attached to a street we found + // Now they are indexed, look for a house attached to a street we found $sHouseNumberRegex = '\\\\m'.$aSearch['sHouseNumber'].'\\\\M'; $sSQL = "select place_id from placex where parent_place_id in (".$sPlaceIDs.") and transliteration(housenumber) ~* E'".$sHouseNumberRegex."'"; if (sizeof($this->aExcludePlaceIDs)) diff --git a/lib/PlaceLookup.php b/lib/PlaceLookup.php index 79c905d3..be60ff07 100644 --- a/lib/PlaceLookup.php +++ b/lib/PlaceLookup.php @@ -142,9 +142,9 @@ } else if ($this->sType == 'interpolation') { - $sSQL = "select place_id, partition, 'I' as osm_type, osm_id, 'place' as class, 'house' as type, null admin_level, housenumber, null as street, null as isin, postcode,"; + $sSQL = "select place_id, partition, 'W' as osm_type, osm_id, 'place' as class, 'house' as type, null admin_level, housenumber, null as street, null as isin, postcode,"; $sSQL .= " calculated_country_code as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,"; - $sSQL .= " coalesce(null,0.75-(30::float/40)) as importance, null as indexed_status, null as indexed_date, null as wikipedia, calculated_country_code, "; + $sSQL .= " (0.75-(30::float/40)) as importance, null as indexed_status, null as indexed_date, null as wikipedia, calculated_country_code, "; $sSQL .= " get_address_by_language(place_id, housenumber, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= " null as placename,"; $sSQL .= " null as ref,"; @@ -187,7 +187,7 @@ if ($this->bAddressDetails) { - if(CONST_Use_US_Tiger_Data && $this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed + if(CONST_Use_US_Tiger_Data && $this->sType == 'tiger' || $this->sType == 'interpolation') // to get addressdetails for tiger data, the housenumber is needed $aAddress = $this->getAddressNames($aPlace['housenumber']); else $aAddress = $this->getAddressNames(); diff --git a/lib/ReverseGeocode.php b/lib/ReverseGeocode.php index 30e74234..05093af3 100644 --- a/lib/ReverseGeocode.php +++ b/lib/ReverseGeocode.php @@ -158,43 +158,15 @@ $bIsInUnitedStates = ($aPlace['calculated_country_code'] == 'us'); } // if a street or house was found, look in interpolation lines table - if ($iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 || $aPlace['rank_search'] == 30)) + if ($iMaxRank_orig >= 28 && $aPlace && $aPlace['rank_search'] >= 26) { - $fSearchDiam = 0.001; - if ($aPlace['rank_search'] == 30) - { - // if a house was found, the closest road needs to be searched, to use its place id as parent_place_id for the interpolation line search - // because a road can be closer to the point than the house from above - $iRoadID = null; - while(!$iRoadID && $fSearchDiam < $fMaxAreaDistance) - { - $fSearchDiam = $fSearchDiam * 2; - $sSQL = 'select place_id '; - $sSQL .= ' FROM placex'; - $sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')'; - $sSQL .= ' and (rank_search = 26 or rank_search = 27)'; - $sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\',\'man_made\')'; - $sSQL .= ' and indexed_status = 0 '; - $sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1'; - $aPlaceRoad = $this->oDB->getRow($sSQL); - if (PEAR::IsError($aPlace)) - { - failInternalError("Could not determine closest place.", $sSQL, $aPlace); - } - $iRoadID = $aPlaceRoad['place_id']; - $iTempPlaceID = $iRoadID; - } - } - else - { - // if a street was found, we can take its place_id as parent_place_id - $iTempPlaceID = $iPlaceID; - } + // if a house was found, search the interpolation line that is at least as close as the house $sSQL = 'SELECT place_id, parent_place_id, 30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction'; - //if (CONST_Debug) { $sSQL .= ', housenumber, ST_distance('.$sPointSQL.', centroid) as distance, st_y(centroid) as lat, st_x(centroid) as lon'; } - $sSQL .= ' FROM location_property_osmline WHERE parent_place_id = '.$iTempPlaceID; - $sSQL .= ' AND ST_DWithin('.$sPointSQL.', linegeo, '.$fSearchDiam.') AND indexed_status = 0'; + $sSQL .= ' FROM location_property_osmline'; + $sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', linegeo, '.$fSearchDiam.')'; + $sSQL .= ' and indexed_status = 0 '; $sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', linegeo) ASC limit 1'; + if (CONST_Debug) { $sSQL = preg_replace('/limit 1/', 'limit 100', $sSQL); @@ -211,7 +183,6 @@ { failInternalError("Could not determine closest housenumber on an osm interpolation line.", $sSQL, $aPlaceLine); } - $iInterpolationLinePlaceID = $aPlaceLine['place_id']; if ($aPlaceLine) { if (CONST_Debug) var_dump('found housenumber in interpolation lines table', $aPlaceLine); @@ -229,7 +200,7 @@ $fDistancePlacex = $aDistancePlacex['distance']; // distance between point and interpolated house (fraction on interpolation line) $sSQL = 'SELECT ST_distance('.$sPointSQL.', ST_LineInterpolatePoint(linegeo, '.$aPlaceLine['fraction'].')) as distance'; - $sSQL .= ' FROM location_property_osmline WHERE place_id = '.$iInterpolationLinePlaceID; + $sSQL .= ' FROM location_property_osmline WHERE place_id = '.$aPlaceLine['place_id']; $aDistanceInterpolation = $this->oDB->getRow($sSQL); if (PEAR::IsError($aDistanceInterpolation)) { @@ -241,7 +212,7 @@ // interpolation is closer to point than placex house $bPlaceIsLine = true; $aPlace = $aPlaceLine; - $iPlaceID = $iInterpolationLinePlaceID; + $iPlaceID = $aPlaceLine['place_id']; $iParentPlaceID = $aPlaceLine['parent_place_id']; // the street $fFraction = $aPlaceLine['fraction']; } diff --git a/nominatim/index.c b/nominatim/index.c index 253b4f20..2a5fa78b 100644 --- a/nominatim/index.c +++ b/nominatim/index.c @@ -21,34 +21,243 @@ extern int verbose; -void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile) +void run_indexing(int rank, int interpolation, PGconn *conn, int num_threads, +struct index_thread_data * thread_data, const char *structuredoutputfile) { - struct index_thread_data * thread_data; - pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER; int tuples, count, sleepcount; - + pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER; + time_t rankStartTime; int rankTotalTuples; int rankCountTuples; float rankPerSecond; - - PGconn *conn; - PGresult * res; + PGresult * resSectors; PGresult * resPlaces; PGresult * resNULL; - - int rank; + int i; int iSector; int iResult; - + const char *paramValues[2]; int paramLengths[2]; int paramFormats[2]; uint32_t paramRank; uint32_t paramSector; uint32_t sector; + + xmlTextWriterPtr writer; + pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER; + + // Create the output file + writer = NULL; + if (structuredoutputfile) + { + writer = nominatim_exportXMLStart(structuredoutputfile); + } + + if (interpolation) + { + fprintf(stderr, "Starting interpolation lines (location_property_osmline)\n"); + } + else + { + fprintf(stderr, "Starting rank %d\n", rank); + } + + rankCountTuples = 0; + rankPerSecond = 0; + + paramRank = PGint32(rank); + paramValues[0] = (char *)¶mRank; + paramLengths[0] = sizeof(paramRank); + paramFormats[0] = 1; + + if (interpolation) + { + resSectors = PQexecPrepared(conn, "index_sectors_osmline", 0, NULL, 0, NULL, 1); + } + else + { + resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); + } + if (PQresultStatus(resSectors) != PGRES_TUPLES_OK) + { + fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn)); + PQclear(resSectors); + exit(EXIT_FAILURE); + } + if (PQftype(resSectors, 0) != PG_OID_INT4) + { + fprintf(stderr, "Sector value has unexpected type\n"); + PQclear(resSectors); + exit(EXIT_FAILURE); + } + if (PQftype(resSectors, 1) != PG_OID_INT8) + { + fprintf(stderr, "Sector value has unexpected type\n"); + PQclear(resSectors); + exit(EXIT_FAILURE); + } + + rankTotalTuples = 0; + for (iSector = 0; iSector < PQntuples(resSectors); iSector++) + { + rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))); + } + + rankStartTime = time(0); + for (iSector = 0; iSector <= PQntuples(resSectors); iSector++) + { + if (iSector > 0) + { + resPlaces = PQgetResult(conn); + if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK) + { + fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); + PQclear(resPlaces); + exit(EXIT_FAILURE); + } + if (PQftype(resPlaces, 0) != PG_OID_INT8) + { + fprintf(stderr, "Place_id value has unexpected type\n"); + PQclear(resPlaces); + exit(EXIT_FAILURE); + } + resNULL = PQgetResult(conn); + if (resNULL != NULL) + { + fprintf(stderr, "Unexpected non-null response\n"); + exit(EXIT_FAILURE); + } + } + + if (iSector < PQntuples(resSectors)) + { + sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); +// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); + + // Get all the place_id's for this sector + paramRank = PGint32(rank); + paramValues[0] = (char *)¶mRank; + paramLengths[0] = sizeof(paramRank); + paramFormats[0] = 1; + paramSector = PGint32(sector); + paramValues[1] = (char *)¶mSector; + paramLengths[1] = sizeof(paramSector); + paramFormats[1] = 1; + if (rankTotalTuples-rankCountTuples < num_threads*1000) + { + // no sectors + if (interpolation) + { + iResult = PQsendQueryPrepared(conn, "index_nosector_places_osmline", 0, NULL, 0, NULL, 1); + } + else + { + iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); + } + } + else + { + if (interpolation) + { + iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1); + } + else + { + iResult = PQsendQueryPrepared(conn, "index_sector_places_osmline", 1, paramValues, paramLengths, paramFormats, 1); + } + } + if (!iResult) + { + fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); + PQclear(resPlaces); + exit(EXIT_FAILURE); + } + } + if (iSector > 0) + { + count = 0; + rankPerSecond = 0; + tuples = PQntuples(resPlaces); + + if (tuples > 0) + { + // Spawn threads + for (i = 0; i < num_threads; i++) + { + thread_data[i].res = resPlaces; + thread_data[i].tuples = tuples; + thread_data[i].count = &count; + thread_data[i].count_mutex = &count_mutex; + thread_data[i].writer = writer; + thread_data[i].writer_mutex = &writer_mutex; + if (interpolation) + { + thread_data[i].table = 0; // use interpolations table + } + else + { + thread_data[i].table = 1; // use placex table + } + pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]); + } + + // Monitor threads to give user feedback + sleepcount = 0; + while (count < tuples) + { + usleep(1000); + + // Aim for one update per second + if (sleepcount++ > 500) + { + rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1); + fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond); + sleepcount = 0; + } + } + + // Wait for everything to finish + for (i = 0; i < num_threads; i++) + { + pthread_join(thread_data[i].thread, NULL); + } + + rankCountTuples += tuples; + } + + // Finished sector + rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1); + fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond); + + PQclear(resPlaces); + } + if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors)) + { + iSector = PQntuples(resSectors) - 1; + } + } + // Finished rank + fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED\n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); + + PQclear(resSectors); + + +} + +void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile) +{ + struct index_thread_data * thread_data; + + PGconn *conn; + PGresult * res; + + int rank; + + int i; xmlTextWriterPtr writer; pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -182,324 +391,22 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co nominatim_exportCreatePreparedQueries(thread_data[i].conn); } - // Create the output file - writer = NULL; - if (structuredoutputfile) - { - writer = nominatim_exportXMLStart(structuredoutputfile); - } fprintf(stderr, "Starting indexing rank (%i to %i) using %i threads\n", rank_min, rank_max, num_threads); - // first for the placex table for (rank = rank_min; rank <= rank_max; rank++) { // OSMLINE: do reindexing (=> reparenting) for interpolation lines at rank 30, but before all other objects of rank 30 // reason: houses (rank 30) depend on the updated interpolation line, when reparenting (see placex_update in functions.sql) if (rank == 30) { - fprintf(stderr, "Starting indexing interpolation lines (location_property_osmline)\n"); - rankCountTuples = 0; - rankTotalTuples = 0; - resSectors = PQexecPrepared(conn, "index_sectors_osmline", 0, NULL, 0, NULL, 1); - if (PQresultStatus(resSectors) != PGRES_TUPLES_OK) - { - fprintf(stderr, "index_sectors_osmline: SELECT failed: %s", PQerrorMessage(conn)); - PQclear(resSectors); - exit(EXIT_FAILURE); - } - if (PQftype(resSectors, 0) != PG_OID_INT4) - { - fprintf(stderr, "Sector value has unexpected type\n"); - PQclear(resSectors); - exit(EXIT_FAILURE); - } - if (PQftype(resSectors, 1) != PG_OID_INT8) - { - fprintf(stderr, "Sector value has unexpected type\n"); - PQclear(resSectors); - exit(EXIT_FAILURE); - } - rankStartTime = time(0); - for (iSector = 0; iSector < PQntuples(resSectors); iSector++) - { - rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))); - } - // do it only if tuples with indexed_status > 0 were found in osmline - int nTuples = PQntuples(resSectors); - if (nTuples > 0) - { - for (iSector = 0; iSector <= nTuples; iSector++) - { - if (iSector > 0) - { - resPlaces = PQgetResult(conn); - if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK) - { - fprintf(stderr, "index_sector_places: SELECT failed: %s\n", PQerrorMessage(conn)); - PQclear(resPlaces); - exit(EXIT_FAILURE); - } - if (PQftype(resPlaces, 0) != PG_OID_INT8) - { - fprintf(stderr, "Place_id value has unexpected type\n"); - PQclear(resPlaces); - exit(EXIT_FAILURE); - } - resNULL = PQgetResult(conn); - if (resNULL != NULL) - { - fprintf(stderr, "Unexpected non-null response\n"); - exit(EXIT_FAILURE); - } - } - - if (iSector < nTuples) - { - sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); - // fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); - - // Get all the place_id's for this sector - paramSector = PGint32(sector); - paramValues[0] = (char *)¶mSector; - paramLengths[0] = sizeof(paramSector); - paramFormats[0] = 1; - if (rankTotalTuples-rankCountTuples < num_threads*1000) - { - // no sectors - iResult = PQsendQueryPrepared(conn, "index_nosector_places_osmline", 0, NULL, 0, NULL, 1); - } - else - { - iResult = PQsendQueryPrepared(conn, "index_sector_places_osmline", 1, paramValues, paramLengths, paramFormats, 1); - } - if (!iResult) - { - fprintf(stderr, "index_sector_places_osmline: SELECT failed: %s", PQerrorMessage(conn)); - PQclear(resPlaces); - exit(EXIT_FAILURE); - } - } - if (iSector > 0) - { - count = 0; - rankPerSecond = 0; - tuples = PQntuples(resPlaces); - - if (tuples > 0) - { - // Spawn threads - for (i = 0; i < num_threads; i++) - { - thread_data[i].res = resPlaces; - thread_data[i].tuples = tuples; - thread_data[i].count = &count; - thread_data[i].count_mutex = &count_mutex; - thread_data[i].writer = writer; - thread_data[i].writer_mutex = &writer_mutex; - thread_data[i].table = 0; // use osmline table - pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]); - } - // Monitor threads to give user feedback - sleepcount = 0; - while (count < tuples) - { - usleep(1000); - - // Aim for one update per second - if (sleepcount++ > 500) - { - rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1); - fprintf(stderr, " Done %i in %i @ %f per second - Interpolation Lines ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - (rankCountTuples + count)))/(float)rankPerSecond); - sleepcount = 0; - } - } - - // Wait for everything to finish - for (i = 0; i < num_threads; i++) - { - pthread_join(thread_data[i].thread, NULL); - } - rankCountTuples += tuples; - } - // Finished sector - rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1); - fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond); - PQclear(resPlaces); - } - if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < nTuples) - { - iSector = nTuples - 1; - } - } - PQclear(resSectors); - } - // Finished rank - fprintf(stderr, "\r Done %i tuples in %i seconds- FINISHED\n", rankCountTuples,(int)(difftime(time(0), rankStartTime))); - if (writer) - { - nominatim_exportXMLEnd(writer); - } + run_indexing(rank, 1, conn, num_threads, thread_data, structuredoutputfile); } - fprintf(stderr, "Starting rank %d\n", rank); - rankCountTuples = 0; - rankPerSecond = 0; - - paramRank = PGint32(rank); - paramValues[0] = (char *)¶mRank; - paramLengths[0] = sizeof(paramRank); - paramFormats[0] = 1; -// if (rank < 16) -// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1); -// else - resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); - - if (PQresultStatus(resSectors) != PGRES_TUPLES_OK) - { - fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn)); - PQclear(resSectors); - exit(EXIT_FAILURE); - } - if (PQftype(resSectors, 0) != PG_OID_INT4) - { - fprintf(stderr, "Sector value has unexpected type\n"); - PQclear(resSectors); - exit(EXIT_FAILURE); - } - if (PQftype(resSectors, 1) != PG_OID_INT8) - { - fprintf(stderr, "Sector value has unexpected type\n"); - PQclear(resSectors); - exit(EXIT_FAILURE); - } - - rankTotalTuples = 0; - for (iSector = 0; iSector < PQntuples(resSectors); iSector++) - { - rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))); - } - - rankStartTime = time(0); - - for (iSector = 0; iSector <= PQntuples(resSectors); iSector++) - { - if (iSector > 0) - { - resPlaces = PQgetResult(conn); - if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK) - { - fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); - PQclear(resPlaces); - exit(EXIT_FAILURE); - } - if (PQftype(resPlaces, 0) != PG_OID_INT8) - { - fprintf(stderr, "Place_id value has unexpected type\n"); - PQclear(resPlaces); - exit(EXIT_FAILURE); - } - resNULL = PQgetResult(conn); - if (resNULL != NULL) - { - fprintf(stderr, "Unexpected non-null response\n"); - exit(EXIT_FAILURE); - } - } - - if (iSector < PQntuples(resSectors)) - { - sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); -// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); - - // Get all the place_id's for this sector - paramRank = PGint32(rank); - paramValues[0] = (char *)¶mRank; - paramLengths[0] = sizeof(paramRank); - paramFormats[0] = 1; - paramSector = PGint32(sector); - paramValues[1] = (char *)¶mSector; - paramLengths[1] = sizeof(paramSector); - paramFormats[1] = 1; - if (rankTotalTuples-rankCountTuples < num_threads*1000) - { - iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); - } - else - { - iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1); - } - if (!iResult) - { - fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); - PQclear(resPlaces); - exit(EXIT_FAILURE); - } - } - - if (iSector > 0) - { - count = 0; - rankPerSecond = 0; - tuples = PQntuples(resPlaces); - - if (tuples > 0) - { - // Spawn threads - for (i = 0; i < num_threads; i++) - { - thread_data[i].res = resPlaces; - thread_data[i].tuples = tuples; - thread_data[i].count = &count; - thread_data[i].count_mutex = &count_mutex; - thread_data[i].writer = writer; - thread_data[i].writer_mutex = &writer_mutex; - thread_data[i].table = 1; // use placex table - pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]); - } - - // Monitor threads to give user feedback - sleepcount = 0; - while (count < tuples) - { - usleep(1000); - - // Aim for one update per second - if (sleepcount++ > 500) - { - rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1); - fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond); - sleepcount = 0; - } - } - - // Wait for everything to finish - for (i = 0; i < num_threads; i++) - { - pthread_join(thread_data[i].thread, NULL); - } - - rankCountTuples += tuples; - } - - // Finished sector - rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1); - fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond); - - PQclear(resPlaces); - } - if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors)) - { - iSector = PQntuples(resSectors) - 1; - } - } - // Finished rank - fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); - - PQclear(resSectors); + run_indexing(rank, 0, conn, num_threads, thread_data, structuredoutputfile); } - + - if (rank_max == 30) + if (rank == 30) { // Close all connections for (i = 0; i < num_threads; i++) diff --git a/sql/functions.sql b/sql/functions.sql index 80763eaa..9b095ce7 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -1090,13 +1090,12 @@ BEGIN END IF; IF OLD.indexed_status = 2 and NEW.indexed_status=0 THEN - -- do the reparenting: (finally here, so that ALL places in placex, that are needed for reparenting, are up to date) + -- do the reparenting: (finally here, because ALL places in placex, that are needed for reparenting, need to be up to date) -- (the osm interpolationline in location_property_osmline was marked for reparenting in placex_insert/placex_delete with index_status = 2 -- => index.c: sets index_status back to 0 -- => triggers this function) place_centroid := ST_PointOnSurface(NEW.linegeo); - -- mark descendants for reparenting - UPDATE placex SET indexed_status = 2 WHERE parent_place_id = OLD.place_id; + -- marking descendants for reparenting is not needed, because there are actually no descendants for interpolation lines NEW.parent_place_id = get_interpolation_parent(NEW.osm_id, NEW.street, null, NEW.partition, place_centroid, NEW.linegeo); -- addr_place (3rd param) is not necessarily needed return NEW; END IF; @@ -1156,14 +1155,6 @@ BEGIN END IF; IF NEW.indexed_status != 0 OR OLD.indexed_status = 0 THEN - - -- if a node(=>house), which is part of a interpolation line, changes (e.g. the street attribute) => mark this line for reparenting - -- (already here, because interpolation lines are reindexed before nodes, so in the second call it would be too late) - -- needed for test case features/db/import: Scenario: addr:street added to housenumbers - IF NEW.osm_type='N' and NEW.class='place' and NEW.type='house' THEN - -- Is this node part of an interpolation line? search for it in location_property_osmline and mark the interpolation line for reparenting - update location_property_osmline p set indexed_status = 2 from planet_osm_ways w where p.linegeo && NEW.geometry and p.osm_id = w.id and NEW.osm_id = any(w.nodes); - END IF; RETURN NEW; END IF; @@ -1906,7 +1897,7 @@ BEGIN UPDATE placex set indexed_status = 100 where osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type; -- interpolations are special - IF OLD.class = 'place' and OLD.type = 'houses' THEN + IF OLD.osm_type='W' and OLD.class = 'place' and OLD.type = 'houses' THEN UPDATE location_property_osmline set indexed_status = 100 where osm_id = OLD.osm_id; -- osm_id = wayid (=old.osm_id) END IF; @@ -1931,7 +1922,6 @@ BEGIN --DEBUG: RAISE WARNING '-----------------------------------------------------------------------------------'; --DEBUG: RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry); - RAISE WARNING 'X3366 - place_isnert'; -- filter wrong tupels IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN INSERT INTO import_polygon_error values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.country_code, @@ -1960,12 +1950,6 @@ BEGIN -- To paraphrase, if there isn't an existing item IF existingline.osm_id IS NULL THEN - IF existing.osm_type IS NOT NULL THEN - -- pathological case caused by the triggerless copy into place during initial import - -- force delete even for large areas, it will be reinserted later - UPDATE place set geometry = ST_SetSRID(ST_Point(0,0), 4326) where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type; - DELETE from place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type; - END IF; -- insert new line into location_property_osmline, use function insert_osmline i = insert_osmline(NEW.osm_id, NEW.housenumber, NEW.street, NEW.addr_place, NEW.postcode, NEW.country_code, NEW.geometry); RETURN NEW; @@ -2002,7 +1986,6 @@ BEGIN -- for interpolations invalidate all nodes on the line update placex p set indexed_status = 2 from planet_osm_ways w where w.id = NEW.osm_id and p.osm_type = 'N' and p.osm_id = any(w.nodes); - RAISE WARNING 'X3399 - updated nodes of interpolation line'; RETURN NULL; ELSE -- insert to placex @@ -2170,6 +2153,7 @@ BEGIN admin_level = NEW.admin_level, geometry = NEW.geometry where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type; + IF NEW.class in ('place','boundary') AND NEW.type in ('postcode','postal_code') THEN IF NEW.postcode IS NULL THEN @@ -2195,6 +2179,14 @@ BEGIN indexed_status = 2, geometry = NEW.geometry where place_id = existingplacex.place_id; + + -- if a node(=>house), which is part of a interpolation line, changes (e.g. the street attribute) => mark this line for reparenting + -- (already here, because interpolation lines are reindexed before nodes, so in the second call it would be too late) + -- needed for test case features/db/import: Scenario: addr:street added to housenumbers + IF NEW.osm_type='N' and NEW.class='place' and NEW.type='house' THEN + -- Is this node part of an interpolation line? search for it in location_property_osmline and mark the interpolation line for reparenting + update location_property_osmline p set indexed_status = 2 from planet_osm_ways w where p.linegeo && NEW.geometry and p.osm_id = w.id and NEW.osm_id = any(w.nodes); + END IF; END IF; @@ -2202,7 +2194,7 @@ BEGIN RETURN NULL; END IF; -END; +END; $$ LANGUAGE plpgsql; diff --git a/sql/tables.sql b/sql/tables.sql index 76a4324f..b4aecdd9 100644 --- a/sql/tables.sql +++ b/sql/tables.sql @@ -84,6 +84,7 @@ GRANT SELECT ON location_property_aux TO "{www-user}"; CREATE TABLE location_property_tiger (linegeo GEOMETRY, place_id BIGINT, partition INTEGER, parent_place_id BIGINT, startnumber INTEGER, endnumber INTEGER, interpolationtype TEXT, postcode TEXT); GRANT SELECT ON location_property_tiger TO "{www-user}"; +drop table if exists location_property_osmline; CREATE TABLE location_property_osmline ( linegeo GEOMETRY, place_id BIGINT NOT NULL, @@ -99,9 +100,11 @@ CREATE TABLE location_property_osmline ( calculated_country_code VARCHAR(2), geometry_sector INTEGER, indexed_status INTEGER, - indexed_date TIMESTAMP); -CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline (place_id) {ts:search-index}; -CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline (parent_place_id) {ts:search-index}; + indexed_date TIMESTAMP){ts:search-data}; +CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {ts:search-index}; +CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {ts:search-index}; +CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {ts:address-index}; +CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {ts:search-index}; GRANT SELECT ON location_property_osmline TO "{www-user}"; drop table IF EXISTS search_name; diff --git a/tests/features/api/reverse.feature b/tests/features/api/reverse.feature index 0dcb12bd..c282d73a 100644 --- a/tests/features/api/reverse.feature +++ b/tests/features/api/reverse.feature @@ -18,7 +18,10 @@ Feature: Reverse geocoding | xml | 4 When looking up coordinates 53.9788769,13.0830313 And results contain valid boundingboxes - + + Scenario: Reverse geocoding for odd interpolated housenumber + + Scenario: Reverse geocoding for even interpolated housenumber @Tiger Scenario: TIGER house number diff --git a/tests/features/db/update/interpolation.feature b/tests/features/db/update/interpolation.feature index 4075ba47..ac72f852 100644 --- a/tests/features/db/update/interpolation.feature +++ b/tests/features/db/update/interpolation.feature @@ -37,7 +37,6 @@ Feature: Update of address interpolations | object | parent_place_id | startnumber | endnumber | W10 | W3 | 2 | 6 - @Fail Scenario: addr:street added to housenumbers Given the scene parallel-road And the place nodes