osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-11-29 08:36:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

add logging of broken polygons, improve address export, better diff output in export, bug in address calculation(#3522), reaply #3520, lost in migration

Browse Source
This commit is contained in:
Brian Quinion 2011-02-07 11:13:18 +00:00
parent bb9f0d81c0
commit 3ea1b35b32
8 changed files with 373 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/lib.php
View File

@ -627,7 +627,7 @@
else else
{ {
if (is_bool($xVal)) return $xVal?'true':'false'; if (is_bool($xVal)) return $xVal?'true':'false';
if (is_numeric($xVal)) return $xVal; // if (is_numeric($xVal)) return $xVal;
return '"'.str_replace('>','\\>',str_replace(array("\n","\r"),'\\n',str_replace(array("\n\r","\r\n"),'\\n',str_replace('"','\\"',$xVal)))).'"'; return '"'.str_replace('>','\\>',str_replace(array("\n","\r"),'\\n',str_replace(array("\n\r","\r\n"),'\\n',str_replace('"','\\"',$xVal)))).'"';
} }
} }

236
nominatim/export.c
View File

@ -18,6 +18,8 @@
extern int verbose; extern int verbose;
int mode = 0;
void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile) void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile)
{ {
xmlTextWriterPtr writer; xmlTextWriterPtr writer;
@ -129,7 +131,7 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch
tuples = PQntuples(resPlaces); tuples = PQntuples(resPlaces);
for (i = 0; i < tuples; i++) for (i = 0; i < tuples; i++)
{ {
nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL); nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL, NULL);
rankTotalDone++; rankTotalDone++;
if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000); if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000);
} }
@ -150,7 +152,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
pg_prepare_params[0] = PG_OID_INT8; pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_details", res = PQprepare(conn, "placex_details",
"select osm_type, osm_id, class, type, name, housenumber, country_code, ST_AsText(geometry), admin_level, rank_address, rank_search from placex where place_id = $1", "select placex.osm_type, placex.osm_id, placex.class, placex.type, placex.name, placex.housenumber, placex.country_code, ST_AsText(placex.geometry), placex.admin_level, placex.rank_address, placex.rank_search, placex.parent_place_id, parent.osm_type, parent.osm_id, placex.indexed_status from placex left outer join placex as parent on (placex.parent_place_id = parent.place_id) where placex.place_id = $1",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
@ -224,23 +226,16 @@ xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile)
fprintf(stderr, "xmlTextWriterWriteAttribute failed\n"); fprintf(stderr, "xmlTextWriterWriteAttribute failed\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
{ mode = 0;
fprintf(stderr, "xmlTextWriterStartElement failed\n");
exit(EXIT_FAILURE);
}
return writer; return writer;
} }
void nominatim_exportXMLEnd(xmlTextWriterPtr writer) void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
{ {
// End <add> nominatim_exportEndMode(writer);
if (xmlTextWriterEndElement(writer) < 0)
{
fprintf(stderr, "xmlTextWriterEndElement failed\n");
exit(EXIT_FAILURE);
}
// End <osmStructured> // End <osmStructured>
if (xmlTextWriterEndElement(writer) < 0) if (xmlTextWriterEndElement(writer) < 0)
{ {
@ -255,135 +250,237 @@ void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
xmlFreeTextWriter(writer); xmlFreeTextWriter(writer);
} }
/* void nominatim_exportStartMode(xmlTextWriterPtr writer, int newMode)
* Requirements: the prepared queries must exist
*/
void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex)
{ {
PGresult * res; if (mode == newMode) return;
PGresult * resNames;
PGresult * resAddress;
PGresult * resExtraTags;
int i; nominatim_exportEndMode(writer);
switch(newMode)
{
case 0:
break;
case 1:
if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
{
fprintf(stderr, "xmlTextWriterStartElement failed\n");
exit(EXIT_FAILURE);
}
break;
case 2:
if (xmlTextWriterStartElement(writer, BAD_CAST "update") < 0)
{
fprintf(stderr, "xmlTextWriterStartElement failed\n");
exit(EXIT_FAILURE);
}
break;
case 3:
if (xmlTextWriterStartElement(writer, BAD_CAST "delete") < 0)
{
fprintf(stderr, "xmlTextWriterStartElement failed\n");
exit(EXIT_FAILURE);
}
break;
}
mode = newMode;
}
void nominatim_exportEndMode(xmlTextWriterPtr writer)
{
if (!mode) return;
if (xmlTextWriterEndElement(writer) < 0)
{
fprintf(stderr, "xmlTextWriterEndElement failed\n");
exit(EXIT_FAILURE);
}
}
void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet)
{
const char * paramValues[1]; const char * paramValues[1];
int paramLengths[1]; int paramLengths[1];
int paramFormats[1]; int paramFormats[1];
uint64_t paramPlaceID; uint64_t paramPlaceID;
paramPlaceID = PGint64(place_id); paramPlaceID = PGint64(place_id);
paramValues[0] = (char *)&paramPlaceID; paramValues[0] = (char *)&paramPlaceID;
paramLengths[0] = sizeof(paramPlaceID); paramLengths[0] = sizeof(paramPlaceID);
paramFormats[0] = 1; paramFormats[0] = 1;
res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0); querySet->res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(res) != PGRES_TUPLES_OK) if (PQresultStatus(querySet->res) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "placex_details: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "placex_details: SELECT failed: %s", PQerrorMessage(conn));
PQclear(res); PQclear(querySet->res);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0); querySet->resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(resNames) != PGRES_TUPLES_OK) if (PQresultStatus(querySet->resNames) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "placex_names: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "placex_names: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resNames); PQclear(querySet->resNames);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0); querySet->resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(resAddress) != PGRES_TUPLES_OK) if (PQresultStatus(querySet->resAddress) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "placex_address: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "placex_address: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resAddress); PQclear(querySet->resAddress);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0); querySet->resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(resExtraTags) != PGRES_TUPLES_OK) if (PQresultStatus(querySet->resExtraTags) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resExtraTags); PQclear(querySet->resExtraTags);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
}
if (writer_mutex) pthread_mutex_lock( writer_mutex ); void nominatim_exportFreeQueries(struct export_data * querySet)
{
PQclear(querySet->res);
PQclear(querySet->resNames);
PQclear(querySet->resAddress);
PQclear(querySet->resExtraTags);
}
/*
* Requirements: the prepared queries must exist
*/
void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet)
{
struct export_data querySet;
int i;
nominatim_exportPlaceQueries(place_id, conn, &querySet);
// Add, modify or delete?
if (prevQuerySet)
{
if ((PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "100") == 0) || PQntuples(querySet.res))
{
// Delete
if (writer_mutex) pthread_mutex_lock( writer_mutex );
nominatim_exportStartMode(writer, 3);
xmlTextWriterStartElement(writer, BAD_CAST "feature");
xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 0));
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 3));
xmlTextWriterEndElement(writer);
if (writer_mutex) pthread_mutex_unlock( writer_mutex );
nominatim_exportFreeQueries(&querySet);
return;
}
if (PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "1") == 0)
{
// Add
if (writer_mutex) pthread_mutex_lock( writer_mutex );
nominatim_exportStartMode(writer, 1);
}
else
{
// Update, but only if something has changed
// TODO: detect changes
if (writer_mutex) pthread_mutex_lock( writer_mutex );
nominatim_exportStartMode(writer, 2);
}
}
else
{
// Add
if (writer_mutex) pthread_mutex_lock( writer_mutex );
nominatim_exportStartMode(writer, 1);
}
xmlTextWriterStartElement(writer, BAD_CAST "feature"); xmlTextWriterStartElement(writer, BAD_CAST "feature");
xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id); xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(res, 0, 0)); xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.res, 0, 0));
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(res, 0, 1)); xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.res, 0, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(res, 0, 2)); xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.res, 0, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(res, 0, 3)); xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.res, 0, 3));
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(res, 0, 9)); xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.res, 0, 9));
xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(res, 0, 10)); xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(querySet.res, 0, 10));
xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_place_id", BAD_CAST PQgetvalue(querySet.res, 0, 11));
xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_type", BAD_CAST PQgetvalue(querySet.res, 0, 12));
xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_id", BAD_CAST PQgetvalue(querySet.res, 0, 13));
if (PQntuples(resNames)) if (PQntuples(querySet.resNames))
{ {
xmlTextWriterStartElement(writer, BAD_CAST "names"); xmlTextWriterStartElement(writer, BAD_CAST "names");
for (i = 0; i < PQntuples(resNames); i++) for (i = 0; i < PQntuples(querySet.resNames); i++)
{ {
xmlTextWriterStartElement(writer, BAD_CAST "name"); xmlTextWriterStartElement(writer, BAD_CAST "name");
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resNames, i, 0)); xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resNames, i, 0));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resNames, i, 1)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resNames, i, 1));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
if (PQgetvalue(res, 0, 5) && strlen(PQgetvalue(res, 0, 5))) if (PQgetvalue(querySet.res, 0, 5) && strlen(PQgetvalue(querySet.res, 0, 5)))
{ {
xmlTextWriterStartElement(writer, BAD_CAST "houseNumber"); xmlTextWriterStartElement(writer, BAD_CAST "houseNumber");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 5)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 5));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
if (PQgetvalue(res, 0, 8) && strlen(PQgetvalue(res, 0, 8))) if (PQgetvalue(querySet.res, 0, 8) && strlen(PQgetvalue(querySet.res, 0, 8)))
{ {
xmlTextWriterStartElement(writer, BAD_CAST "adminLevel"); xmlTextWriterStartElement(writer, BAD_CAST "adminLevel");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 8)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 8));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
if (PQgetvalue(res, 0, 6) && strlen(PQgetvalue(res, 0, 6))) if (PQgetvalue(querySet.res, 0, 6) && strlen(PQgetvalue(querySet.res, 0, 6)))
{ {
xmlTextWriterStartElement(writer, BAD_CAST "countryCode"); xmlTextWriterStartElement(writer, BAD_CAST "countryCode");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 6)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 6));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
if (PQntuples(resAddress) > 0) if (PQntuples(querySet.resAddress) > 0)
{ {
xmlTextWriterStartElement(writer, BAD_CAST "address"); xmlTextWriterStartElement(writer, BAD_CAST "address");
for (i = 0; i < PQntuples(resAddress); i++) for (i = 0; i < PQntuples(querySet.resAddress); i++)
{ {
xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(resAddress, i, 5)))); xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(querySet.resAddress, i, 5))));
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(resAddress, i, 5)); xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.resAddress, i, 5));
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resAddress, i, 0)); xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resAddress, i, 0));
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(resAddress, i, 1)); xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.resAddress, i, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(resAddress, i, 2)); xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.resAddress, i, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(resAddress, i, 3)); xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.resAddress, i, 3));
xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(resAddress, i, 4)); xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(querySet.resAddress, i, 4));
xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(resAddress, i, 6)); xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(querySet.resAddress, i, 6));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
if (PQntuples(resExtraTags)) if (PQntuples(querySet.resExtraTags))
{ {
xmlTextWriterStartElement(writer, BAD_CAST "tags"); xmlTextWriterStartElement(writer, BAD_CAST "tags");
for (i = 0; i < PQntuples(resExtraTags); i++) for (i = 0; i < PQntuples(querySet.resExtraTags); i++)
{ {
xmlTextWriterStartElement(writer, BAD_CAST "tag"); xmlTextWriterStartElement(writer, BAD_CAST "tag");
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resExtraTags, i, 0)); xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resExtraTags, i, 0));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resExtraTags, i, 1)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resExtraTags, i, 1));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
@ -392,17 +489,14 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry"); xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 7)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 7));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
xmlTextWriterEndElement(writer); // </feature> xmlTextWriterEndElement(writer); // </feature>
if (writer_mutex) pthread_mutex_unlock( writer_mutex ); if (writer_mutex) pthread_mutex_unlock( writer_mutex );
PQclear(res); nominatim_exportFreeQueries(&querySet);
PQclear(resNames);
PQclear(resAddress);
PQclear(resExtraTags);
} }
const char * getRankLabel(int rank) const char * getRankLabel(int rank)

18
nominatim/export.h
View File

@ -5,11 +5,27 @@
#include <libxml/xmlwriter.h> #include <libxml/xmlwriter.h>
#include <stdint.h> #include <stdint.h>
struct export_data
{
PGresult * res;
PGresult * resNames;
PGresult * resAddress;
PGresult * resExtraTags;
};
void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile); void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile);
void nominatim_exportCreatePreparedQueries(PGconn * conn); void nominatim_exportCreatePreparedQueries(PGconn * conn);
xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile); xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile);
void nominatim_exportXMLEnd(xmlTextWriterPtr writer); void nominatim_exportXMLEnd(xmlTextWriterPtr writer);
void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex);
void nominatim_exportEndMode(xmlTextWriterPtr writer);
void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet);
void nominatim_exportFreeQueries(struct export_data * querySet);
void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet);
const char * getRankLabel(int rank); const char * getRankLabel(int rank);
#endif #endif

100
nominatim/import.c
View File

@ -50,6 +50,9 @@ struct feature
xmlChar * rankAddress; xmlChar * rankAddress;
xmlChar * rankSearch; xmlChar * rankSearch;
xmlChar * countryCode; xmlChar * countryCode;
xmlChar * parentPlaceID;
xmlChar * parentType;
xmlChar * parentID;
xmlChar * adminLevel; xmlChar * adminLevel;
xmlChar * houseNumber; xmlChar * houseNumber;
xmlChar * geometry; xmlChar * geometry;
@ -67,6 +70,7 @@ int featureNameLines = 0;
int featureExtraTagLines = 0; int featureExtraTagLines = 0;
int featureCount = 0; int featureCount = 0;
xmlHashTablePtr partionTableTagsHash; xmlHashTablePtr partionTableTagsHash;
xmlHashTablePtr partionTableTagsHashDelete;
char featureNameString[MAX_FEATURENAMESTRING]; char featureNameString[MAX_FEATURENAMESTRING];
char featureExtraTagString[MAX_FEATUREEXTRATAGSTRING]; char featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
@ -135,13 +139,17 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank"); feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance"); feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
feature.countryCode = NULL; feature.countryCode = NULL;
feature.adminLevel = NULL; feature.adminLevel = NULL;
feature.houseNumber = NULL; feature.houseNumber = NULL;
feature.geometry = NULL; feature.geometry = NULL;
featureAddressLines = 0; featureAddressLines = 0;
featureNameLines = 0; featureNameLines = 0;
featureExtraTagLines = 0; featureExtraTagLines = 0;
return; return;
} }
@ -312,7 +320,6 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
void EndElement(xmlTextReaderPtr reader, const xmlChar *name) void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
{ {
PGresult * res; PGresult * res;
PGresult * resPlaceID;
const char * paramValues[11]; const char * paramValues[11];
char * place_id; char * place_id;
char * partionQueryName; char * partionQueryName;
@ -348,9 +355,9 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
} }
} }
*/ */
place_id = feature.placeID; place_id = (char *)feature.placeID;
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE) if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
{ {
paramValues[0] = (const char *)place_id; paramValues[0] = (const char *)place_id;
res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0); res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
@ -379,6 +386,19 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
if (partionQueryName)
{
res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
PQclear(res);
exit(EXIT_FAILURE);
}
PQclear(res);
}
} }
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD) if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
@ -398,8 +418,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
lineValueLen = 0; lineValueLen = 0;
for (i = 0; i < featureNameLines; i++) for (i = 0; i < featureNameLines; i++)
{ {
lineTypeLen = strlen(BAD_CAST featureName[i].type); lineTypeLen = (int)strlen((char *) featureName[i].type);
lineValueLen = strlen(BAD_CAST featureName[i].value); lineValueLen = (int)strlen((char *) featureName[i].value);
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING) if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
{ {
fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value); fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
@ -407,11 +427,11 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
} }
if (namePos) strcpy(featureNameString+(namePos++), ","); if (namePos) strcpy(featureNameString+(namePos++), ",");
strcpy(featureNameString+(namePos++), "\""); strcpy(featureNameString+(namePos++), "\"");
strcpy(featureNameString+namePos, BAD_CAST featureName[i].type); strcpy(featureNameString+namePos, (char*) featureName[i].type);
namePos += lineTypeLen; namePos += lineTypeLen;
strcpy(featureNameString+namePos, "\"=>\""); strcpy(featureNameString+namePos, "\"=>\"");
namePos += 4; namePos += 4;
strcpy(featureNameString+namePos, BAD_CAST featureName[i].value); strcpy(featureNameString+namePos, (char *) featureName[i].value);
namePos += lineValueLen; namePos += lineValueLen;
strcpy(featureNameString+(namePos++), "\""); strcpy(featureNameString+(namePos++), "\"");
} }
@ -426,8 +446,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
lineValueLen = 0; lineValueLen = 0;
for (i = 0; i < featureExtraTagLines; i++) for (i = 0; i < featureExtraTagLines; i++)
{ {
lineTypeLen = strlen(BAD_CAST featureExtraTag[i].type); lineTypeLen = strlen((char *) featureExtraTag[i].type);
lineValueLen = strlen(BAD_CAST featureExtraTag[i].value); lineValueLen = strlen((char *) featureExtraTag[i].value);
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING) if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
{ {
fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value); fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
@ -435,30 +455,36 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
} }
if (namePos) strcpy(featureExtraTagString+(namePos++),","); if (namePos) strcpy(featureExtraTagString+(namePos++),",");
strcpy(featureExtraTagString+(namePos++), "\""); strcpy(featureExtraTagString+(namePos++), "\"");
strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].type); strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
namePos += lineTypeLen; namePos += lineTypeLen;
strcpy(featureExtraTagString+namePos, "\"=>\""); strcpy(featureExtraTagString+namePos, "\"=>\"");
namePos += 4; namePos += 4;
strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].value); strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
namePos += lineValueLen; namePos += lineValueLen;
strcpy(featureExtraTagString+(namePos++), "\""); strcpy(featureExtraTagString+(namePos++), "\"");
} }
} }
paramValues[6] = (const char *)featureExtraTagString; paramValues[6] = (const char *)featureExtraTagString;
paramValues[7] = (const char *)feature.adminLevel; paramValues[7] = (const char *)feature.parentPlaceID;
paramValues[8] = (const char *)feature.houseNumber;
paramValues[9] = (const char *)feature.rankAddress; paramValues[8] = (const char *)feature.adminLevel;
paramValues[10] = (const char *)feature.rankSearch; paramValues[9] = (const char *)feature.houseNumber;
paramValues[11] = (const char *)feature.geometry; paramValues[10] = (const char *)feature.rankAddress;
res = PQexecPrepared(conn, "placex_insert", 12, paramValues, NULL, NULL, 0); paramValues[11] = (const char *)feature.rankSearch;
if (PQresultStatus(res) != PGRES_COMMAND_OK) paramValues[12] = (const char *)feature.geometry;
if (strlen(paramValues[3]))
{ {
fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn)); res = PQexecPrepared(conn, "placex_insert", 13, paramValues, NULL, NULL, 0);
PQclear(res); if (PQresultStatus(res) != PGRES_COMMAND_OK)
exit(EXIT_FAILURE); {
fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
PQclear(res);
exit(EXIT_FAILURE);
}
PQclear(res);
} }
PQclear(res);
for (i = 0; i < featureAddressLines; i++) for (i = 0; i < featureAddressLines; i++)
{ {
@ -513,7 +539,6 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
} }
} }
@ -595,6 +620,7 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
} }
partionTableTagsHash = xmlHashCreate(200); partionTableTagsHash = xmlHashCreate(200);
partionTableTagsHashDelete = xmlHashCreate(200);
partionTagsFile = fopen(partionTagsFilename, "rt"); partionTagsFile = fopen(partionTagsFilename, "rt");
if (!partionTagsFile) if (!partionTagsFile)
@ -636,6 +662,27 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
} }
xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName); xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
strcpy(partionQueryName, "partition_delete_");
strcat(partionQueryName, osmkey);
strcat(partionQueryName, "_");
strcat(partionQueryName, osmvalue);
strcpy(partionQuerySQL, "delete from place_classtype_");
strcat(partionQuerySQL, osmkey);
strcat(partionQuerySQL, "_");
strcat(partionQuerySQL, osmvalue);
strcat(partionQuerySQL, " where place_id = $1::integer");
res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
} }
res = PQprepare(conn, "get_new_place_id", res = PQprepare(conn, "get_new_place_id",
@ -657,8 +704,8 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
} }
res = PQprepare(conn, "placex_insert", res = PQprepare(conn, "placex_insert",
"insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,admin_level,housenumber,rank_address,rank_search,geometry) " "insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
"values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, st_setsrid($12, 4326))", "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, st_setsrid($13, 4326))",
12, NULL); 12, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
@ -740,6 +787,7 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
xmlFreeTextReader(reader); xmlFreeTextReader(reader);
xmlHashFree(partionTableTagsHash, NULL); xmlHashFree(partionTableTagsHash, NULL);
xmlHashFree(partionTableTagsHashDelete, NULL);
return 0; return 0;
} }

25
nominatim/index.c
View File

@ -40,6 +40,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
int i; int i;
int iSector; int iSector;
int iResult; int iResult;
int bSkip;
const char *paramValues[2]; const char *paramValues[2];
int paramLengths[2]; int paramLengths[2];
@ -160,7 +161,8 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
// if (rank < 16) // if (rank < 16)
// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1); // resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
// else // else
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK) if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
@ -215,7 +217,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
if (iSector < PQntuples(resSectors)) if (iSector < PQntuples(resSectors))
{ {
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
//printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); // printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
// Get all the place_id's for this sector // Get all the place_id's for this sector
paramRank = PGint32(rank); paramRank = PGint32(rank);
@ -226,9 +228,9 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
paramValues[1] = (char *)&paramSector; paramValues[1] = (char *)&paramSector;
paramLengths[1] = sizeof(paramSector); paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1; paramFormats[1] = 1;
if (rankTotalTuples-rankCountTuples < num_threads*20) if (rankTotalTuples-rankCountTuples < num_threads*1000)
{ {
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
} }
else else
{ {
@ -292,7 +294,10 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
PQclear(resPlaces); PQclear(resPlaces);
} }
if (rankTotalTuples-rankCountTuples < num_threads*20) iSector = PQntuples(resSectors); if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
{
iSector = PQntuples(resSectors) - 1;
}
} }
// Finished rank // Finished rank
printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
@ -309,6 +314,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
void *nominatim_indexThread(void * thread_data_in) void *nominatim_indexThread(void * thread_data_in)
{ {
struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in; struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in;
struct export_data querySet;
PGresult *res; PGresult *res;
@ -339,6 +345,12 @@ void *nominatim_indexThread(void * thread_data_in)
int done = 0; int done = 0;
while(!done) while(!done)
{ {
if (thread_data->writer)
{
nominatim_exportPlaceQueries(place_id, thread_data->conn, &querySet);
}
paramPlaceID = PGint32(place_id); paramPlaceID = PGint32(place_id);
paramValues[0] = (char *)&paramPlaceID; paramValues[0] = (char *)&paramPlaceID;
paramLengths[0] = sizeof(paramPlaceID); paramLengths[0] = sizeof(paramPlaceID);
@ -366,7 +378,8 @@ void *nominatim_indexThread(void * thread_data_in)
if (thread_data->writer) if (thread_data->writer)
{ {
nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex); nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex, &querySet);
nominatim_exportFreeQueries(&querySet);
} }
} }

109
sql/functions.sql
View File

@ -1102,7 +1102,7 @@ BEGIN
-- RETURN NULL; -- RETURN NULL;
-- END IF; -- END IF;
-- RETURN NEW; -- The following is not needed until doing diff updates, and slows the main index process down RETURN NEW; -- The following is not needed until doing diff updates, and slows the main index process down
IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
-- Performance: We just can't handle re-indexing for country level changes -- Performance: We just can't handle re-indexing for country level changes
@ -1218,6 +1218,12 @@ BEGIN
result := deleteSearchName(NEW.partition, NEW.place_id); result := deleteSearchName(NEW.partition, NEW.place_id);
DELETE FROM place_addressline WHERE place_id = NEW.place_id; DELETE FROM place_addressline WHERE place_id = NEW.place_id;
DELETE FROM place_boundingbox where place_id = NEW.place_id; DELETE FROM place_boundingbox where place_id = NEW.place_id;
result := deleteRoad(NEW.partition, NEW.place_id);
result := deleteLocationArea(NEW.partition, NEW.place_id);
NEW.country_code := lower(get_country_code(NEW.geometry, NEW.country_code));
NEW.partition := get_partition(NEW.geometry, NEW.country_code);
NEW.geometry_sector := geometry_sector(NEW.partition, NEW.geometry);
-- Adding ourselves to the list simplifies address calculations later -- Adding ourselves to the list simplifies address calculations later
INSERT INTO place_addressline VALUES (NEW.place_id, NEW.place_id, true, true, 0, NEW.rank_address); INSERT INTO place_addressline VALUES (NEW.place_id, NEW.place_id, true, true, 0, NEW.rank_address);
@ -1345,7 +1351,7 @@ BEGIN
END LOOP; END LOOP;
END IF; END IF;
--RAISE WARNING 'x3'; --RAISE WARNING 'x3 %',NEW.parent_place_id;
IF NEW.parent_place_id IS NULL AND NEW.street IS NOT NULL THEN IF NEW.parent_place_id IS NULL AND NEW.street IS NOT NULL THEN
address_street_word_id := get_name_id(make_standard_name(NEW.street)); address_street_word_id := get_name_id(make_standard_name(NEW.street));
@ -1356,11 +1362,13 @@ BEGIN
END IF; END IF;
END IF; END IF;
--RAISE WARNING 'x4'; --RAISE WARNING 'x4 %',NEW.parent_place_id;
-- Still nothing, just use the nearest road -- Still nothing, just use the nearest road
FOR location IN SELECT place_id FROM getNearestRoadFeature(NEW.partition, place_centroid) LOOP IF NEW.parent_place_id IS NULL THEN
NEW.parent_place_id := location.place_id; FOR location IN SELECT place_id FROM getNearestRoadFeature(NEW.partition, place_centroid) LOOP
END LOOP; NEW.parent_place_id := location.place_id;
END LOOP;
END IF;
--return NEW; --return NEW;
--RAISE WARNING 'x6 %',NEW.parent_place_id; --RAISE WARNING 'x6 %',NEW.parent_place_id;
@ -1548,6 +1556,10 @@ BEGIN
DELETE FROM place_addressline where address_place_id = OLD.place_id; DELETE FROM place_addressline where address_place_id = OLD.place_id;
b := deleteRoad(OLD.partition, OLD.place_id);
update placex set indexed_status = 2 where parent_place_id = OLD.place_id and indexed_status = 0;
END IF; END IF;
IF OLD.rank_address < 26 THEN IF OLD.rank_address < 26 THEN
@ -1596,7 +1608,7 @@ DECLARE
partition INTEGER; partition INTEGER;
BEGIN BEGIN
IF FALSE AND NEW.osm_type = 'R' THEN IF FALSE and NEW.osm_type = 'R' THEN
RAISE WARNING '-----------------------------------------------------------------------------------'; RAISE WARNING '-----------------------------------------------------------------------------------';
RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry); RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry);
select * from placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type INTO existingplacex; select * from placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type INTO existingplacex;
@ -1612,6 +1624,8 @@ BEGIN
END IF; END IF;
IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN
INSERT INTO import_polygon_error values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.country_code,
now(), ST_IsValidReason(NEW.geometry), null, NEW.geometry);
-- RAISE WARNING 'Invalid Geometry: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type; -- RAISE WARNING 'Invalid Geometry: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
RETURN null; RETURN null;
END IF; END IF;
@ -1629,13 +1643,26 @@ BEGIN
-- Handle a place changing type by removing the old data -- Handle a place changing type by removing the old data
-- My generated 'place' types are causing havok because they overlap with real tags -- My generated 'place' types are causing havok because they overlap with real tags
-- TODO: move them to their own special purpose tag to avoid collisions -- TODO: move them to their own special purpose key/class to avoid collisions
IF existing.osm_type IS NULL AND (NEW.type not in ('postcode','house','houses')) THEN -- IF existing.osm_type IS NULL AND (NEW.type not in ('postcode','house','houses')) THEN
DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type not in ('postcode','house','houses'); -- DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type not in ('postcode','house','houses');
END IF; -- END IF;
-- RAISE WARNING 'Existing: %',existing.place_id; -- RAISE WARNING 'Existing: %',existing.place_id;
-- Log and discard
IF existing.geometry is not null AND st_isvalid(existing.geometry)
AND st_area(existing.geometry) > 0.02
AND ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon')
AND st_area(NEW.geometry) < st_area(existing.geometry)*0.5
THEN
INSERT INTO import_polygon_error values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.country_code, now(),
'Area reduced from '||st_area(existing.geometry)||' to '||st_area(NEW.geometry), existing.geometry, NEW.geometry);
RETURN null;
END IF;
DELETE from import_polygon_error where osm_type = NEW.osm_type and osm_id = NEW.osm_id;
-- To paraphrase, if there isn't an existing item, OR if the admin level has changed, OR if it is a major change in geometry -- To paraphrase, if there isn't an existing item, OR if the admin level has changed, OR if it is a major change in geometry
IF existing.osm_type IS NULL IF existing.osm_type IS NULL
OR existingplacex.osm_type IS NULL OR existingplacex.osm_type IS NULL
@ -1652,7 +1679,6 @@ BEGIN
-- RAISE WARNING 'no existing placex %', existingplacex; -- RAISE WARNING 'no existing placex %', existingplacex;
-- END IF; -- END IF;
-- RAISE WARNING 'delete and replace'; -- RAISE WARNING 'delete and replace';
IF existing.osm_type IS NOT NULL THEN IF existing.osm_type IS NOT NULL THEN
@ -1719,7 +1745,7 @@ BEGIN
select geometry from placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type into existinggeometry; select geometry from placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type into existinggeometry;
-- Performance limit -- Performance limit
IF st_area(NEW.geometry) < 1 AND st_area(existinggeometry) < 1 THEN IF st_area(NEW.geometry) < 0.000000001 AND st_area(existinggeometry) < 1 THEN
-- re-index points that have moved in / out of the polygon, could be done as a single query but postgres gets the index usage wrong -- re-index points that have moved in / out of the polygon, could be done as a single query but postgres gets the index usage wrong
update placex set indexed_status = 2 where indexed_status = 0 and update placex set indexed_status = 2 where indexed_status = 0 and
@ -1767,8 +1793,8 @@ BEGIN
-- performance, can't take the load of re-indexing a whole country / huge area -- performance, can't take the load of re-indexing a whole country / huge area
IF st_area(NEW.geometry) < 0.5 THEN IF st_area(NEW.geometry) < 0.5 THEN
UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = existingplacex.place_id -- UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = existingplacex.place_id
and placex.place_id = place_addressline.place_id and indexed_status = 0; -- and placex.place_id = place_addressline.place_id and indexed_status = 0;
END IF; END IF;
END IF; END IF;
@ -1809,7 +1835,7 @@ BEGIN
geometry = NEW.geometry geometry = NEW.geometry
where place_id = existingplacex.place_id; where place_id = existingplacex.place_id;
-- now done as part of indexing -- now done as part of insert
-- partition := get_partition(NEW.geometry, existingplacex.country_code); -- partition := get_partition(NEW.geometry, existingplacex.country_code);
-- result := update_location(partition, existingplacex.place_id, existingplacex.country_code, NEW.name, existingplacex.rank_search, existingplacex.rank_address, NEW.geometry); -- result := update_location(partition, existingplacex.place_id, existingplacex.country_code, NEW.name, existingplacex.rank_search, existingplacex.rank_address, NEW.geometry);
@ -1939,6 +1965,7 @@ create type addressline as (
name HSTORE, name HSTORE,
class TEXT, class TEXT,
type TEXT, type TEXT,
admin_level INTEGER,
fromarea BOOLEAN, fromarea BOOLEAN,
isaddress BOOLEAN, isaddress BOOLEAN,
rank_address INTEGER, rank_address INTEGER,
@ -1953,28 +1980,32 @@ DECLARE
search TEXT[]; search TEXT[];
found INTEGER; found INTEGER;
location RECORD; location RECORD;
countrylocation RECORD;
searchcountrycode varchar(2); searchcountrycode varchar(2);
searchhousenumber TEXT; searchhousenumber TEXT;
searchhousename HSTORE; searchhousename HSTORE;
searchrankaddress INTEGER; searchrankaddress INTEGER;
searchpostcode TEXT; searchpostcode TEXT;
searchclass TEXT;
searchtype TEXT;
countryname HSTORE; countryname HSTORE;
hadcountry BOOLEAN;
BEGIN BEGIN
select parent_place_id,'us', housenumber, 30, postcode, null from location_property_tiger select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_tiger
WHERE place_id = in_place_id WHERE place_id = in_place_id
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename; INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
IF for_place_id IS NULL THEN IF for_place_id IS NULL THEN
select parent_place_id,'us', housenumber, 30, postcode, null from location_property_aux select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux
WHERE place_id = in_place_id WHERE place_id = in_place_id
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename; INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
END IF; END IF;
IF for_place_id IS NULL THEN IF for_place_id IS NULL THEN
select parent_place_id, country_code, housenumber, rank_address, postcode, name from placex select parent_place_id, country_code, housenumber, rank_address, postcode, name, class, type from placex
WHERE place_id = in_place_id and rank_address = 30 WHERE place_id = in_place_id and rank_address = 30
INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename; INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
END IF; END IF;
IF for_place_id IS NULL THEN IF for_place_id IS NULL THEN
@ -1986,12 +2017,13 @@ BEGIN
--RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode; --RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode;
found := 1000; found := 1000;
hadcountry := false;
FOR location IN FOR location IN
select placex.place_id, osm_type, osm_id, select placex.place_id, osm_type, osm_id,
CASE WHEN class = 'place' and type = 'postcode' THEN 'name' => postcode ELSE name END as name, CASE WHEN class = 'place' and type = 'postcode' THEN 'name' => postcode ELSE name END as name,
class, type, fromarea, isaddress, class, type, admin_level, fromarea, isaddress,
CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address, CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address,
distance distance,country_code
from place_addressline join placex on (address_place_id = placex.place_id) from place_addressline join placex on (address_place_id = placex.place_id)
where place_addressline.place_id = for_place_id where place_addressline.place_id = for_place_id
and ((cached_rank_address > 0 AND cached_rank_address < searchrankaddress) OR address_place_id = for_place_id) and ((cached_rank_address > 0 AND cached_rank_address < searchrankaddress) OR address_place_id = for_place_id)
@ -1999,10 +2031,26 @@ BEGIN
order by rank_address desc,isaddress desc,fromarea desc,distance asc,rank_search desc order by rank_address desc,isaddress desc,fromarea desc,distance asc,rank_search desc
LOOP LOOP
--RAISE WARNING '%',location; --RAISE WARNING '%',location;
IF searchcountrycode IS NULL AND location.country_code IS NOT NULL THEN
searchcountrycode := location.country_code;
END IF;
IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN
location.isaddress := FALSE; location.isaddress := FALSE;
END IF; END IF;
RETURN NEXT location; IF location.rank_address = 4 AND location.isaddress THEN
hadcountry := true;
END IF;
IF location.rank_address < 4 AND NOT hadcountry THEN
select name from country_name where country_code = searchcountrycode limit 1 INTO countryname;
IF countryname IS NOT NULL THEN
countrylocation := ROW(null, null, null, countryname, 'place', 'country', null, true, true, 4, 0)::addressline;
RETURN NEXT countrylocation;
END IF;
END IF;
countrylocation := ROW(location.place_id, location.osm_type, location.osm_id, location.name, location.class,
location.type, location.admin_level, location.fromarea, location.isaddress, location.rank_address,
location.distance)::addressline;
RETURN NEXT countrylocation;
found := location.rank_address; found := location.rank_address;
END LOOP; END LOOP;
@ -2010,28 +2058,29 @@ BEGIN
select name from country_name where country_code = searchcountrycode limit 1 INTO countryname; select name from country_name where country_code = searchcountrycode limit 1 INTO countryname;
--RAISE WARNING '% % %',found,searchcountrycode,countryname; --RAISE WARNING '% % %',found,searchcountrycode,countryname;
IF countryname IS NOT NULL THEN IF countryname IS NOT NULL THEN
location := ROW(null, null, null, countryname, 'place', 'country', true, true, 4, 0)::addressline; location := ROW(null, null, null, countryname, 'place', 'country', null, true, true, 4, 0)::addressline;
RETURN NEXT location; RETURN NEXT location;
END IF; END IF;
END IF; END IF;
IF searchcountrycode IS NOT NULL THEN IF searchcountrycode IS NOT NULL THEN
location := ROW(null, null, null, 'ref'=>searchcountrycode, 'place', 'country_code', true, false, 4, 0)::addressline; location := ROW(null, null, null, 'ref'=>searchcountrycode, 'place', 'country_code', null, true, false, 4, 0)::addressline;
RETURN NEXT location; RETURN NEXT location;
END IF; END IF;
IF searchhousename IS NOT NULL THEN IF searchhousename IS NOT NULL THEN
location := ROW(in_place_id, null, null, searchhousename, 'place', 'house_name', true, true, 29, 0)::addressline; location := ROW(in_place_id, null, null, searchhousename, searchclass, searchtype, null, true, true, 29, 0)::addressline;
-- location := ROW(in_place_id, null, null, searchhousename, 'place', 'house_name', null, true, true, 29, 0)::addressline;
RETURN NEXT location; RETURN NEXT location;
END IF; END IF;
IF searchhousenumber IS NOT NULL THEN IF searchhousenumber IS NOT NULL THEN
location := ROW(in_place_id, null, null, 'ref'=>searchhousenumber, 'place', 'house_number', true, true, 28, 0)::addressline; location := ROW(in_place_id, null, null, 'ref'=>searchhousenumber, 'place', 'house_number', null, true, true, 28, 0)::addressline;
RETURN NEXT location; RETURN NEXT location;
END IF; END IF;
IF searchpostcode IS NOT NULL THEN IF searchpostcode IS NOT NULL THEN
location := ROW(null, null, null, 'ref'=>searchpostcode, 'place', 'postcode', true, true, 5, 0)::addressline; location := ROW(null, null, null, 'ref'=>searchpostcode, 'place', 'postcode', null, true, true, 5, 0)::addressline;
RETURN NEXT location; RETURN NEXT location;
END IF; END IF;

15
sql/tables.sql
View File

@ -274,3 +274,18 @@ CREATE INDEX idx_placex_sector ON placex USING BTREE (geometry_sector,rank_addre
DROP SEQUENCE seq_postcodes; DROP SEQUENCE seq_postcodes;
CREATE SEQUENCE seq_postcodes start 1; CREATE SEQUENCE seq_postcodes start 1;
drop table import_polygon_error;
CREATE TABLE import_polygon_error (
osm_type char(1),
osm_id INTEGER,
class TEXT NOT NULL,
type TEXT NOT NULL,
name HSTORE,
country_code varchar(2),
updated timestamp,
errormessage text
);
SELECT AddGeometryColumn('import_polygon_error', 'prevgeometry', 4326, 'GEOMETRY', 2);
SELECT AddGeometryColumn('import_polygon_error', 'newgeometry', 4326, 'GEOMETRY', 2);
CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id);

10
utils/setup.php
View File

@ -130,13 +130,9 @@
{ {
echo "Partitions\n"; echo "Partitions\n";
$bDidSomething = true; $bDidSomething = true;
echo "here";
$oDB =& getDB(); $oDB =& getDB();
echo "there";
$sSQL = 'select partition from country_name order by country_code'; $sSQL = 'select partition from country_name order by country_code';
var_dump($sSQL);
$aPartitions = $oDB->getCol($sSQL); $aPartitions = $oDB->getCol($sSQL);
var_dump($aPartitions);
if (PEAR::isError($aPartitions)) if (PEAR::isError($aPartitions))
{ {
fail($aPartitions->getMessage()); fail($aPartitions->getMessage());
@ -147,11 +143,9 @@ var_dump($aPartitions);
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
foreach($aMatches as $aMatch) foreach($aMatches as $aMatch)
{ {
var_dump($aMatch);
$sResult = ''; $sResult = '';
foreach($aPartitions as $sPartitionName) foreach($aPartitions as $sPartitionName)
{ {
var_dump($sPartitionName);
$sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
} }
$sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
@ -371,8 +365,8 @@ var_dump($sPartitionName);
{ {
// Convert database DSN to psql paramaters // Convert database DSN to psql paramaters
$aDSNInfo = DB::parseDSN(CONST_Database_DSN); $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
$sCMD = 'psql '.$aDSNInfo['database']; if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
$sCMD = 'psql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database'];
$aDescriptors = array( $aDescriptors = array(
0 => array('pipe', 'r'), 0 => array('pipe', 'r'),
1 => array('pipe', 'w'), 1 => array('pipe', 'w'),