mirror of
https://github.com/osm-search/Nominatim.git
synced 2024-11-25 19:35:02 +03:00
add logging of broken polygons, improve address export, better diff output in export, bug in address calculation(#3522), reaply #3520, lost in migration
This commit is contained in:
Brian Quinion
parent
bb9f0d81c0
commit
3ea1b35b32
@ -627,7 +627,7 @@
|
||||
else
|
||||
{
|
||||
if (is_bool($xVal)) return $xVal?'true':'false';
|
||||
if (is_numeric($xVal)) return $xVal;
|
||||
// if (is_numeric($xVal)) return $xVal;
|
||||
return '"'.str_replace('>','\\>',str_replace(array("\n","\r"),'\\n',str_replace(array("\n\r","\r\n"),'\\n',str_replace('"','\\"',$xVal)))).'"';
|
||||
}
|
||||
}
|
||||
|
||||
@ -18,6 +18,8 @@
|
||||
|
||||
extern int verbose;
|
||||
|
||||
int mode = 0;
|
||||
|
||||
void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile)
|
||||
{
|
||||
xmlTextWriterPtr writer;
|
||||
@ -129,7 +131,7 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch
|
||||
tuples = PQntuples(resPlaces);
|
||||
for (i = 0; i < tuples; i++)
|
||||
{
|
||||
nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL);
|
||||
nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL, NULL);
|
||||
rankTotalDone++;
|
||||
if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000);
|
||||
}
|
||||
@ -150,7 +152,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
|
||||
|
||||
pg_prepare_params[0] = PG_OID_INT8;
|
||||
res = PQprepare(conn, "placex_details",
|
||||
"select osm_type, osm_id, class, type, name, housenumber, country_code, ST_AsText(geometry), admin_level, rank_address, rank_search from placex where place_id = $1",
|
||||
"select placex.osm_type, placex.osm_id, placex.class, placex.type, placex.name, placex.housenumber, placex.country_code, ST_AsText(placex.geometry), placex.admin_level, placex.rank_address, placex.rank_search, placex.parent_place_id, parent.osm_type, parent.osm_id, placex.indexed_status from placex left outer join placex as parent on (placex.parent_place_id = parent.place_id) where placex.place_id = $1",
|
||||
1, pg_prepare_params);
|
||||
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
||||
{
|
||||
@ -224,23 +226,16 @@ xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile)
|
||||
fprintf(stderr, "xmlTextWriterWriteAttribute failed\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
|
||||
{
|
||||
fprintf(stderr, "xmlTextWriterStartElement failed\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
mode = 0;
|
||||
|
||||
return writer;
|
||||
}
|
||||
|
||||
void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
|
||||
{
|
||||
// End <add>
|
||||
if (xmlTextWriterEndElement(writer) < 0)
|
||||
{
|
||||
fprintf(stderr, "xmlTextWriterEndElement failed\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
nominatim_exportEndMode(writer);
|
||||
|
||||
// End <osmStructured>
|
||||
if (xmlTextWriterEndElement(writer) < 0)
|
||||
{
|
||||
@ -255,135 +250,237 @@ void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
|
||||
xmlFreeTextWriter(writer);
|
||||
}
|
||||
|
||||
/*
|
||||
* Requirements: the prepared queries must exist
|
||||
*/
|
||||
void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex)
|
||||
void nominatim_exportStartMode(xmlTextWriterPtr writer, int newMode)
|
||||
{
|
||||
PGresult * res;
|
||||
PGresult * resNames;
|
||||
PGresult * resAddress;
|
||||
PGresult * resExtraTags;
|
||||
if (mode == newMode) return;
|
||||
|
||||
int i;
|
||||
nominatim_exportEndMode(writer);
|
||||
|
||||
switch(newMode)
|
||||
{
|
||||
case 0:
|
||||
break;
|
||||
|
||||
case 1:
|
||||
if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
|
||||
{
|
||||
fprintf(stderr, "xmlTextWriterStartElement failed\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
break;
|
||||
|
||||
case 2:
|
||||
if (xmlTextWriterStartElement(writer, BAD_CAST "update") < 0)
|
||||
{
|
||||
fprintf(stderr, "xmlTextWriterStartElement failed\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
break;
|
||||
|
||||
case 3:
|
||||
if (xmlTextWriterStartElement(writer, BAD_CAST "delete") < 0)
|
||||
{
|
||||
fprintf(stderr, "xmlTextWriterStartElement failed\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
break;
|
||||
}
|
||||
mode = newMode;
|
||||
}
|
||||
|
||||
void nominatim_exportEndMode(xmlTextWriterPtr writer)
|
||||
{
|
||||
if (!mode) return;
|
||||
|
||||
if (xmlTextWriterEndElement(writer) < 0)
|
||||
{
|
||||
fprintf(stderr, "xmlTextWriterEndElement failed\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet)
|
||||
{
|
||||
const char * paramValues[1];
|
||||
int paramLengths[1];
|
||||
int paramFormats[1];
|
||||
uint64_t paramPlaceID;
|
||||
|
||||
|
||||
paramPlaceID = PGint64(place_id);
|
||||
paramValues[0] = (char *)¶mPlaceID;
|
||||
paramLengths[0] = sizeof(paramPlaceID);
|
||||
paramFormats[0] = 1;
|
||||
|
||||
res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0);
|
||||
if (PQresultStatus(res) != PGRES_TUPLES_OK)
|
||||
querySet->res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0);
|
||||
if (PQresultStatus(querySet->res) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "placex_details: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(res);
|
||||
PQclear(querySet->res);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0);
|
||||
if (PQresultStatus(resNames) != PGRES_TUPLES_OK)
|
||||
querySet->resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0);
|
||||
if (PQresultStatus(querySet->resNames) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "placex_names: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resNames);
|
||||
PQclear(querySet->resNames);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0);
|
||||
if (PQresultStatus(resAddress) != PGRES_TUPLES_OK)
|
||||
querySet->resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0);
|
||||
if (PQresultStatus(querySet->resAddress) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "placex_address: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resAddress);
|
||||
PQclear(querySet->resAddress);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
|
||||
if (PQresultStatus(resExtraTags) != PGRES_TUPLES_OK)
|
||||
querySet->resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
|
||||
if (PQresultStatus(querySet->resExtraTags) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resExtraTags);
|
||||
PQclear(querySet->resExtraTags);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (writer_mutex) pthread_mutex_lock( writer_mutex );
|
||||
void nominatim_exportFreeQueries(struct export_data * querySet)
|
||||
{
|
||||
PQclear(querySet->res);
|
||||
PQclear(querySet->resNames);
|
||||
PQclear(querySet->resAddress);
|
||||
PQclear(querySet->resExtraTags);
|
||||
}
|
||||
|
||||
/*
|
||||
* Requirements: the prepared queries must exist
|
||||
*/
|
||||
void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
|
||||
xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet)
|
||||
{
|
||||
struct export_data querySet;
|
||||
|
||||
int i;
|
||||
|
||||
nominatim_exportPlaceQueries(place_id, conn, &querySet);
|
||||
|
||||
// Add, modify or delete?
|
||||
if (prevQuerySet)
|
||||
{
|
||||
if ((PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "100") == 0) || PQntuples(querySet.res))
|
||||
{
|
||||
// Delete
|
||||
if (writer_mutex) pthread_mutex_lock( writer_mutex );
|
||||
nominatim_exportStartMode(writer, 3);
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "feature");
|
||||
xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 0));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 1));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 2));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 3));
|
||||
xmlTextWriterEndElement(writer);
|
||||
if (writer_mutex) pthread_mutex_unlock( writer_mutex );
|
||||
nominatim_exportFreeQueries(&querySet);
|
||||
return;
|
||||
}
|
||||
if (PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "1") == 0)
|
||||
{
|
||||
// Add
|
||||
if (writer_mutex) pthread_mutex_lock( writer_mutex );
|
||||
nominatim_exportStartMode(writer, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Update, but only if something has changed
|
||||
|
||||
// TODO: detect changes
|
||||
|
||||
if (writer_mutex) pthread_mutex_lock( writer_mutex );
|
||||
nominatim_exportStartMode(writer, 2);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Add
|
||||
if (writer_mutex) pthread_mutex_lock( writer_mutex );
|
||||
nominatim_exportStartMode(writer, 1);
|
||||
}
|
||||
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "feature");
|
||||
xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(res, 0, 0));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(res, 0, 1));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(res, 0, 2));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(res, 0, 3));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(res, 0, 9));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(res, 0, 10));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.res, 0, 0));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.res, 0, 1));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.res, 0, 2));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.res, 0, 3));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.res, 0, 9));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(querySet.res, 0, 10));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_place_id", BAD_CAST PQgetvalue(querySet.res, 0, 11));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_type", BAD_CAST PQgetvalue(querySet.res, 0, 12));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_id", BAD_CAST PQgetvalue(querySet.res, 0, 13));
|
||||
|
||||
if (PQntuples(resNames))
|
||||
if (PQntuples(querySet.resNames))
|
||||
{
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "names");
|
||||
|
||||
for (i = 0; i < PQntuples(resNames); i++)
|
||||
for (i = 0; i < PQntuples(querySet.resNames); i++)
|
||||
{
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "name");
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resNames, i, 0));
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resNames, i, 1));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resNames, i, 0));
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resNames, i, 1));
|
||||
xmlTextWriterEndElement(writer);
|
||||
}
|
||||
|
||||
xmlTextWriterEndElement(writer);
|
||||
}
|
||||
|
||||
if (PQgetvalue(res, 0, 5) && strlen(PQgetvalue(res, 0, 5)))
|
||||
if (PQgetvalue(querySet.res, 0, 5) && strlen(PQgetvalue(querySet.res, 0, 5)))
|
||||
{
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "houseNumber");
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 5));
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 5));
|
||||
xmlTextWriterEndElement(writer);
|
||||
}
|
||||
|
||||
if (PQgetvalue(res, 0, 8) && strlen(PQgetvalue(res, 0, 8)))
|
||||
if (PQgetvalue(querySet.res, 0, 8) && strlen(PQgetvalue(querySet.res, 0, 8)))
|
||||
{
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "adminLevel");
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 8));
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 8));
|
||||
xmlTextWriterEndElement(writer);
|
||||
}
|
||||
|
||||
if (PQgetvalue(res, 0, 6) && strlen(PQgetvalue(res, 0, 6)))
|
||||
if (PQgetvalue(querySet.res, 0, 6) && strlen(PQgetvalue(querySet.res, 0, 6)))
|
||||
{
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "countryCode");
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 6));
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 6));
|
||||
xmlTextWriterEndElement(writer);
|
||||
}
|
||||
|
||||
if (PQntuples(resAddress) > 0)
|
||||
if (PQntuples(querySet.resAddress) > 0)
|
||||
{
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "address");
|
||||
for (i = 0; i < PQntuples(resAddress); i++)
|
||||
for (i = 0; i < PQntuples(querySet.resAddress); i++)
|
||||
{
|
||||
xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(resAddress, i, 5))));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(resAddress, i, 5));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resAddress, i, 0));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(resAddress, i, 1));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(resAddress, i, 2));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(resAddress, i, 3));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(resAddress, i, 4));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(resAddress, i, 6));
|
||||
xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(querySet.resAddress, i, 5))));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.resAddress, i, 5));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resAddress, i, 0));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.resAddress, i, 1));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.resAddress, i, 2));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.resAddress, i, 3));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(querySet.resAddress, i, 4));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(querySet.resAddress, i, 6));
|
||||
xmlTextWriterEndElement(writer);
|
||||
}
|
||||
xmlTextWriterEndElement(writer);
|
||||
}
|
||||
|
||||
if (PQntuples(resExtraTags))
|
||||
if (PQntuples(querySet.resExtraTags))
|
||||
{
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "tags");
|
||||
|
||||
for (i = 0; i < PQntuples(resExtraTags); i++)
|
||||
for (i = 0; i < PQntuples(querySet.resExtraTags); i++)
|
||||
{
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "tag");
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resExtraTags, i, 0));
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resExtraTags, i, 1));
|
||||
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resExtraTags, i, 0));
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resExtraTags, i, 1));
|
||||
xmlTextWriterEndElement(writer);
|
||||
}
|
||||
|
||||
@ -392,17 +489,14 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
|
||||
|
||||
|
||||
xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry");
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 7));
|
||||
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 7));
|
||||
xmlTextWriterEndElement(writer);
|
||||
|
||||
xmlTextWriterEndElement(writer); // </feature>
|
||||
|
||||
if (writer_mutex) pthread_mutex_unlock( writer_mutex );
|
||||
|
||||
PQclear(res);
|
||||
PQclear(resNames);
|
||||
PQclear(resAddress);
|
||||
PQclear(resExtraTags);
|
||||
nominatim_exportFreeQueries(&querySet);
|
||||
}
|
||||
|
||||
const char * getRankLabel(int rank)
|
||||
|
||||
@ -5,11 +5,27 @@
|
||||
#include <libxml/xmlwriter.h>
|
||||
#include <stdint.h>
|
||||
|
||||
struct export_data
|
||||
{
|
||||
PGresult * res;
|
||||
PGresult * resNames;
|
||||
PGresult * resAddress;
|
||||
PGresult * resExtraTags;
|
||||
};
|
||||
|
||||
void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile);
|
||||
void nominatim_exportCreatePreparedQueries(PGconn * conn);
|
||||
|
||||
xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile);
|
||||
void nominatim_exportXMLEnd(xmlTextWriterPtr writer);
|
||||
void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex);
|
||||
|
||||
void nominatim_exportEndMode(xmlTextWriterPtr writer);
|
||||
|
||||
void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet);
|
||||
void nominatim_exportFreeQueries(struct export_data * querySet);
|
||||
|
||||
void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
|
||||
xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet);
|
||||
const char * getRankLabel(int rank);
|
||||
|
||||
#endif
|
||||
|
||||
@ -50,6 +50,9 @@ struct feature
|
||||
xmlChar * rankAddress;
|
||||
xmlChar * rankSearch;
|
||||
xmlChar * countryCode;
|
||||
xmlChar * parentPlaceID;
|
||||
xmlChar * parentType;
|
||||
xmlChar * parentID;
|
||||
xmlChar * adminLevel;
|
||||
xmlChar * houseNumber;
|
||||
xmlChar * geometry;
|
||||
@ -67,6 +70,7 @@ int featureNameLines = 0;
|
||||
int featureExtraTagLines = 0;
|
||||
int featureCount = 0;
|
||||
xmlHashTablePtr partionTableTagsHash;
|
||||
xmlHashTablePtr partionTableTagsHashDelete;
|
||||
char featureNameString[MAX_FEATURENAMESTRING];
|
||||
char featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
|
||||
|
||||
@ -135,13 +139,17 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
|
||||
feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
|
||||
feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
|
||||
|
||||
feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
|
||||
feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
|
||||
feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
|
||||
|
||||
feature.countryCode = NULL;
|
||||
feature.adminLevel = NULL;
|
||||
feature.houseNumber = NULL;
|
||||
feature.geometry = NULL;
|
||||
featureAddressLines = 0;
|
||||
featureNameLines = 0;
|
||||
featureExtraTagLines = 0;
|
||||
featureExtraTagLines = 0;
|
||||
|
||||
return;
|
||||
}
|
||||
@ -312,7 +320,6 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
|
||||
void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
|
||||
{
|
||||
PGresult * res;
|
||||
PGresult * resPlaceID;
|
||||
const char * paramValues[11];
|
||||
char * place_id;
|
||||
char * partionQueryName;
|
||||
@ -348,9 +355,9 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
|
||||
}
|
||||
}
|
||||
*/
|
||||
place_id = feature.placeID;
|
||||
place_id = (char *)feature.placeID;
|
||||
|
||||
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE)
|
||||
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
|
||||
{
|
||||
paramValues[0] = (const char *)place_id;
|
||||
res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
|
||||
@ -379,6 +386,19 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
PQclear(res);
|
||||
|
||||
partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
|
||||
if (partionQueryName)
|
||||
{
|
||||
res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
|
||||
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
||||
{
|
||||
fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
|
||||
PQclear(res);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
PQclear(res);
|
||||
}
|
||||
}
|
||||
|
||||
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
|
||||
@ -398,8 +418,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
|
||||
lineValueLen = 0;
|
||||
for (i = 0; i < featureNameLines; i++)
|
||||
{
|
||||
lineTypeLen = strlen(BAD_CAST featureName[i].type);
|
||||
lineValueLen = strlen(BAD_CAST featureName[i].value);
|
||||
lineTypeLen = (int)strlen((char *) featureName[i].type);
|
||||
lineValueLen = (int)strlen((char *) featureName[i].value);
|
||||
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
|
||||
{
|
||||
fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
|
||||
@ -407,11 +427,11 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
|
||||
}
|
||||
if (namePos) strcpy(featureNameString+(namePos++), ",");
|
||||
strcpy(featureNameString+(namePos++), "\"");
|
||||
strcpy(featureNameString+namePos, BAD_CAST featureName[i].type);
|
||||
strcpy(featureNameString+namePos, (char*) featureName[i].type);
|
||||
namePos += lineTypeLen;
|
||||
strcpy(featureNameString+namePos, "\"=>\"");
|
||||
namePos += 4;
|
||||
strcpy(featureNameString+namePos, BAD_CAST featureName[i].value);
|
||||
strcpy(featureNameString+namePos, (char *) featureName[i].value);
|
||||
namePos += lineValueLen;
|
||||
strcpy(featureNameString+(namePos++), "\"");
|
||||
}
|
||||
@ -426,8 +446,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
|
||||
lineValueLen = 0;
|
||||
for (i = 0; i < featureExtraTagLines; i++)
|
||||
{
|
||||
lineTypeLen = strlen(BAD_CAST featureExtraTag[i].type);
|
||||
lineValueLen = strlen(BAD_CAST featureExtraTag[i].value);
|
||||
lineTypeLen = strlen((char *) featureExtraTag[i].type);
|
||||
lineValueLen = strlen((char *) featureExtraTag[i].value);
|
||||
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
|
||||
{
|
||||
fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
|
||||
@ -435,30 +455,36 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
|
||||
}
|
||||
if (namePos) strcpy(featureExtraTagString+(namePos++),",");
|
||||
strcpy(featureExtraTagString+(namePos++), "\"");
|
||||
strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].type);
|
||||
strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
|
||||
namePos += lineTypeLen;
|
||||
strcpy(featureExtraTagString+namePos, "\"=>\"");
|
||||
namePos += 4;
|
||||
strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].value);
|
||||
strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
|
||||
namePos += lineValueLen;
|
||||
strcpy(featureExtraTagString+(namePos++), "\"");
|
||||
}
|
||||
}
|
||||
paramValues[6] = (const char *)featureExtraTagString;
|
||||
|
||||
paramValues[7] = (const char *)feature.adminLevel;
|
||||
paramValues[8] = (const char *)feature.houseNumber;
|
||||
paramValues[9] = (const char *)feature.rankAddress;
|
||||
paramValues[10] = (const char *)feature.rankSearch;
|
||||
paramValues[11] = (const char *)feature.geometry;
|
||||
res = PQexecPrepared(conn, "placex_insert", 12, paramValues, NULL, NULL, 0);
|
||||
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
||||
paramValues[7] = (const char *)feature.parentPlaceID;
|
||||
|
||||
paramValues[8] = (const char *)feature.adminLevel;
|
||||
paramValues[9] = (const char *)feature.houseNumber;
|
||||
paramValues[10] = (const char *)feature.rankAddress;
|
||||
paramValues[11] = (const char *)feature.rankSearch;
|
||||
paramValues[12] = (const char *)feature.geometry;
|
||||
if (strlen(paramValues[3]))
|
||||
{
|
||||
fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(res);
|
||||
exit(EXIT_FAILURE);
|
||||
res = PQexecPrepared(conn, "placex_insert", 13, paramValues, NULL, NULL, 0);
|
||||
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
||||
{
|
||||
fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
|
||||
fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
|
||||
PQclear(res);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
PQclear(res);
|
||||
}
|
||||
PQclear(res);
|
||||
|
||||
for (i = 0; i < featureAddressLines; i++)
|
||||
{
|
||||
@ -513,7 +539,6 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
PQclear(res);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -595,6 +620,7 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
|
||||
}
|
||||
|
||||
partionTableTagsHash = xmlHashCreate(200);
|
||||
partionTableTagsHashDelete = xmlHashCreate(200);
|
||||
|
||||
partionTagsFile = fopen(partionTagsFilename, "rt");
|
||||
if (!partionTagsFile)
|
||||
@ -636,6 +662,27 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
|
||||
}
|
||||
|
||||
xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
|
||||
|
||||
partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
|
||||
strcpy(partionQueryName, "partition_delete_");
|
||||
strcat(partionQueryName, osmkey);
|
||||
strcat(partionQueryName, "_");
|
||||
strcat(partionQueryName, osmvalue);
|
||||
|
||||
strcpy(partionQuerySQL, "delete from place_classtype_");
|
||||
strcat(partionQuerySQL, osmkey);
|
||||
strcat(partionQuerySQL, "_");
|
||||
strcat(partionQuerySQL, osmvalue);
|
||||
strcat(partionQuerySQL, " where place_id = $1::integer");
|
||||
|
||||
res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
|
||||
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
||||
{
|
||||
fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
|
||||
}
|
||||
|
||||
res = PQprepare(conn, "get_new_place_id",
|
||||
@ -657,8 +704,8 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
|
||||
}
|
||||
|
||||
res = PQprepare(conn, "placex_insert",
|
||||
"insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,admin_level,housenumber,rank_address,rank_search,geometry) "
|
||||
"values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, st_setsrid($12, 4326))",
|
||||
"insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
|
||||
"values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, st_setsrid($13, 4326))",
|
||||
12, NULL);
|
||||
if (PQresultStatus(res) != PGRES_COMMAND_OK)
|
||||
{
|
||||
@ -740,6 +787,7 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
|
||||
|
||||
xmlFreeTextReader(reader);
|
||||
xmlHashFree(partionTableTagsHash, NULL);
|
||||
xmlHashFree(partionTableTagsHashDelete, NULL);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -40,6 +40,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
|
||||
int i;
|
||||
int iSector;
|
||||
int iResult;
|
||||
int bSkip;
|
||||
|
||||
const char *paramValues[2];
|
||||
int paramLengths[2];
|
||||
@ -160,7 +161,8 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
|
||||
// if (rank < 16)
|
||||
// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
// else
|
||||
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
|
||||
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
|
||||
@ -215,7 +217,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
|
||||
if (iSector < PQntuples(resSectors))
|
||||
{
|
||||
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
|
||||
//printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
|
||||
// printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
|
||||
|
||||
// Get all the place_id's for this sector
|
||||
paramRank = PGint32(rank);
|
||||
@ -226,9 +228,9 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
|
||||
paramValues[1] = (char *)¶mSector;
|
||||
paramLengths[1] = sizeof(paramSector);
|
||||
paramFormats[1] = 1;
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*20)
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*1000)
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -292,7 +294,10 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
|
||||
|
||||
PQclear(resPlaces);
|
||||
}
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*20) iSector = PQntuples(resSectors);
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
|
||||
{
|
||||
iSector = PQntuples(resSectors) - 1;
|
||||
}
|
||||
}
|
||||
// Finished rank
|
||||
printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
|
||||
@ -309,6 +314,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
|
||||
void *nominatim_indexThread(void * thread_data_in)
|
||||
{
|
||||
struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in;
|
||||
struct export_data querySet;
|
||||
|
||||
PGresult *res;
|
||||
|
||||
@ -339,6 +345,12 @@ void *nominatim_indexThread(void * thread_data_in)
|
||||
int done = 0;
|
||||
while(!done)
|
||||
{
|
||||
if (thread_data->writer)
|
||||
{
|
||||
nominatim_exportPlaceQueries(place_id, thread_data->conn, &querySet);
|
||||
}
|
||||
|
||||
|
||||
paramPlaceID = PGint32(place_id);
|
||||
paramValues[0] = (char *)¶mPlaceID;
|
||||
paramLengths[0] = sizeof(paramPlaceID);
|
||||
@ -366,7 +378,8 @@ void *nominatim_indexThread(void * thread_data_in)
|
||||
|
||||
if (thread_data->writer)
|
||||
{
|
||||
nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex);
|
||||
nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex, &querySet);
|
||||
nominatim_exportFreeQueries(&querySet);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1102,7 +1102,7 @@ BEGIN
|
||||
-- RETURN NULL;
|
||||
-- END IF;
|
||||
|
||||
-- RETURN NEW; -- The following is not needed until doing diff updates, and slows the main index process down
|
||||
RETURN NEW; -- The following is not needed until doing diff updates, and slows the main index process down
|
||||
|
||||
IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
|
||||
-- Performance: We just can't handle re-indexing for country level changes
|
||||
@ -1218,6 +1218,12 @@ BEGIN
|
||||
result := deleteSearchName(NEW.partition, NEW.place_id);
|
||||
DELETE FROM place_addressline WHERE place_id = NEW.place_id;
|
||||
DELETE FROM place_boundingbox where place_id = NEW.place_id;
|
||||
result := deleteRoad(NEW.partition, NEW.place_id);
|
||||
result := deleteLocationArea(NEW.partition, NEW.place_id);
|
||||
|
||||
NEW.country_code := lower(get_country_code(NEW.geometry, NEW.country_code));
|
||||
NEW.partition := get_partition(NEW.geometry, NEW.country_code);
|
||||
NEW.geometry_sector := geometry_sector(NEW.partition, NEW.geometry);
|
||||
|
||||
-- Adding ourselves to the list simplifies address calculations later
|
||||
INSERT INTO place_addressline VALUES (NEW.place_id, NEW.place_id, true, true, 0, NEW.rank_address);
|
||||
@ -1345,7 +1351,7 @@ BEGIN
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
--RAISE WARNING 'x3';
|
||||
--RAISE WARNING 'x3 %',NEW.parent_place_id;
|
||||
|
||||
IF NEW.parent_place_id IS NULL AND NEW.street IS NOT NULL THEN
|
||||
address_street_word_id := get_name_id(make_standard_name(NEW.street));
|
||||
@ -1356,11 +1362,13 @@ BEGIN
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
--RAISE WARNING 'x4';
|
||||
--RAISE WARNING 'x4 %',NEW.parent_place_id;
|
||||
-- Still nothing, just use the nearest road
|
||||
FOR location IN SELECT place_id FROM getNearestRoadFeature(NEW.partition, place_centroid) LOOP
|
||||
NEW.parent_place_id := location.place_id;
|
||||
END LOOP;
|
||||
IF NEW.parent_place_id IS NULL THEN
|
||||
FOR location IN SELECT place_id FROM getNearestRoadFeature(NEW.partition, place_centroid) LOOP
|
||||
NEW.parent_place_id := location.place_id;
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
--return NEW;
|
||||
--RAISE WARNING 'x6 %',NEW.parent_place_id;
|
||||
@ -1548,6 +1556,10 @@ BEGIN
|
||||
|
||||
DELETE FROM place_addressline where address_place_id = OLD.place_id;
|
||||
|
||||
b := deleteRoad(OLD.partition, OLD.place_id);
|
||||
|
||||
update placex set indexed_status = 2 where parent_place_id = OLD.place_id and indexed_status = 0;
|
||||
|
||||
END IF;
|
||||
|
||||
IF OLD.rank_address < 26 THEN
|
||||
@ -1596,7 +1608,7 @@ DECLARE
|
||||
partition INTEGER;
|
||||
BEGIN
|
||||
|
||||
IF FALSE AND NEW.osm_type = 'R' THEN
|
||||
IF FALSE and NEW.osm_type = 'R' THEN
|
||||
RAISE WARNING '-----------------------------------------------------------------------------------';
|
||||
RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry);
|
||||
select * from placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type INTO existingplacex;
|
||||
@ -1612,6 +1624,8 @@ BEGIN
|
||||
END IF;
|
||||
|
||||
IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN
|
||||
INSERT INTO import_polygon_error values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.country_code,
|
||||
now(), ST_IsValidReason(NEW.geometry), null, NEW.geometry);
|
||||
-- RAISE WARNING 'Invalid Geometry: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
|
||||
RETURN null;
|
||||
END IF;
|
||||
@ -1629,13 +1643,26 @@ BEGIN
|
||||
|
||||
-- Handle a place changing type by removing the old data
|
||||
-- My generated 'place' types are causing havok because they overlap with real tags
|
||||
-- TODO: move them to their own special purpose tag to avoid collisions
|
||||
IF existing.osm_type IS NULL AND (NEW.type not in ('postcode','house','houses')) THEN
|
||||
DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type not in ('postcode','house','houses');
|
||||
END IF;
|
||||
-- TODO: move them to their own special purpose key/class to avoid collisions
|
||||
-- IF existing.osm_type IS NULL AND (NEW.type not in ('postcode','house','houses')) THEN
|
||||
-- DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type not in ('postcode','house','houses');
|
||||
-- END IF;
|
||||
|
||||
-- RAISE WARNING 'Existing: %',existing.place_id;
|
||||
|
||||
-- Log and discard
|
||||
IF existing.geometry is not null AND st_isvalid(existing.geometry)
|
||||
AND st_area(existing.geometry) > 0.02
|
||||
AND ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon')
|
||||
AND st_area(NEW.geometry) < st_area(existing.geometry)*0.5
|
||||
THEN
|
||||
INSERT INTO import_polygon_error values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.country_code, now(),
|
||||
'Area reduced from '||st_area(existing.geometry)||' to '||st_area(NEW.geometry), existing.geometry, NEW.geometry);
|
||||
RETURN null;
|
||||
END IF;
|
||||
|
||||
DELETE from import_polygon_error where osm_type = NEW.osm_type and osm_id = NEW.osm_id;
|
||||
|
||||
-- To paraphrase, if there isn't an existing item, OR if the admin level has changed, OR if it is a major change in geometry
|
||||
IF existing.osm_type IS NULL
|
||||
OR existingplacex.osm_type IS NULL
|
||||
@ -1652,7 +1679,6 @@ BEGIN
|
||||
-- RAISE WARNING 'no existing placex %', existingplacex;
|
||||
-- END IF;
|
||||
|
||||
|
||||
-- RAISE WARNING 'delete and replace';
|
||||
|
||||
IF existing.osm_type IS NOT NULL THEN
|
||||
@ -1719,7 +1745,7 @@ BEGIN
|
||||
select geometry from placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type into existinggeometry;
|
||||
|
||||
-- Performance limit
|
||||
IF st_area(NEW.geometry) < 1 AND st_area(existinggeometry) < 1 THEN
|
||||
IF st_area(NEW.geometry) < 0.000000001 AND st_area(existinggeometry) < 1 THEN
|
||||
|
||||
-- re-index points that have moved in / out of the polygon, could be done as a single query but postgres gets the index usage wrong
|
||||
update placex set indexed_status = 2 where indexed_status = 0 and
|
||||
@ -1767,8 +1793,8 @@ BEGIN
|
||||
|
||||
-- performance, can't take the load of re-indexing a whole country / huge area
|
||||
IF st_area(NEW.geometry) < 0.5 THEN
|
||||
UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = existingplacex.place_id
|
||||
and placex.place_id = place_addressline.place_id and indexed_status = 0;
|
||||
-- UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = existingplacex.place_id
|
||||
-- and placex.place_id = place_addressline.place_id and indexed_status = 0;
|
||||
END IF;
|
||||
|
||||
END IF;
|
||||
@ -1809,7 +1835,7 @@ BEGIN
|
||||
geometry = NEW.geometry
|
||||
where place_id = existingplacex.place_id;
|
||||
|
||||
-- now done as part of indexing
|
||||
-- now done as part of insert
|
||||
-- partition := get_partition(NEW.geometry, existingplacex.country_code);
|
||||
-- result := update_location(partition, existingplacex.place_id, existingplacex.country_code, NEW.name, existingplacex.rank_search, existingplacex.rank_address, NEW.geometry);
|
||||
|
||||
@ -1939,6 +1965,7 @@ create type addressline as (
|
||||
name HSTORE,
|
||||
class TEXT,
|
||||
type TEXT,
|
||||
admin_level INTEGER,
|
||||
fromarea BOOLEAN,
|
||||
isaddress BOOLEAN,
|
||||
rank_address INTEGER,
|
||||
@ -1953,28 +1980,32 @@ DECLARE
|
||||
search TEXT[];
|
||||
found INTEGER;
|
||||
location RECORD;
|
||||
countrylocation RECORD;
|
||||
searchcountrycode varchar(2);
|
||||
searchhousenumber TEXT;
|
||||
searchhousename HSTORE;
|
||||
searchrankaddress INTEGER;
|
||||
searchpostcode TEXT;
|
||||
searchclass TEXT;
|
||||
searchtype TEXT;
|
||||
countryname HSTORE;
|
||||
hadcountry BOOLEAN;
|
||||
BEGIN
|
||||
|
||||
select parent_place_id,'us', housenumber, 30, postcode, null from location_property_tiger
|
||||
select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_tiger
|
||||
WHERE place_id = in_place_id
|
||||
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
|
||||
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
|
||||
|
||||
IF for_place_id IS NULL THEN
|
||||
select parent_place_id,'us', housenumber, 30, postcode, null from location_property_aux
|
||||
select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux
|
||||
WHERE place_id = in_place_id
|
||||
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
|
||||
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
|
||||
END IF;
|
||||
|
||||
IF for_place_id IS NULL THEN
|
||||
select parent_place_id, country_code, housenumber, rank_address, postcode, name from placex
|
||||
select parent_place_id, country_code, housenumber, rank_address, postcode, name, class, type from placex
|
||||
WHERE place_id = in_place_id and rank_address = 30
|
||||
INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
|
||||
INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
|
||||
END IF;
|
||||
|
||||
IF for_place_id IS NULL THEN
|
||||
@ -1986,12 +2017,13 @@ BEGIN
|
||||
--RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode;
|
||||
|
||||
found := 1000;
|
||||
hadcountry := false;
|
||||
FOR location IN
|
||||
select placex.place_id, osm_type, osm_id,
|
||||
CASE WHEN class = 'place' and type = 'postcode' THEN 'name' => postcode ELSE name END as name,
|
||||
class, type, fromarea, isaddress,
|
||||
class, type, admin_level, fromarea, isaddress,
|
||||
CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address,
|
||||
distance
|
||||
distance,country_code
|
||||
from place_addressline join placex on (address_place_id = placex.place_id)
|
||||
where place_addressline.place_id = for_place_id
|
||||
and ((cached_rank_address > 0 AND cached_rank_address < searchrankaddress) OR address_place_id = for_place_id)
|
||||
@ -1999,10 +2031,26 @@ BEGIN
|
||||
order by rank_address desc,isaddress desc,fromarea desc,distance asc,rank_search desc
|
||||
LOOP
|
||||
--RAISE WARNING '%',location;
|
||||
IF searchcountrycode IS NULL AND location.country_code IS NOT NULL THEN
|
||||
searchcountrycode := location.country_code;
|
||||
END IF;
|
||||
IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN
|
||||
location.isaddress := FALSE;
|
||||
END IF;
|
||||
RETURN NEXT location;
|
||||
IF location.rank_address = 4 AND location.isaddress THEN
|
||||
hadcountry := true;
|
||||
END IF;
|
||||
IF location.rank_address < 4 AND NOT hadcountry THEN
|
||||
select name from country_name where country_code = searchcountrycode limit 1 INTO countryname;
|
||||
IF countryname IS NOT NULL THEN
|
||||
countrylocation := ROW(null, null, null, countryname, 'place', 'country', null, true, true, 4, 0)::addressline;
|
||||
RETURN NEXT countrylocation;
|
||||
END IF;
|
||||
END IF;
|
||||
countrylocation := ROW(location.place_id, location.osm_type, location.osm_id, location.name, location.class,
|
||||
location.type, location.admin_level, location.fromarea, location.isaddress, location.rank_address,
|
||||
location.distance)::addressline;
|
||||
RETURN NEXT countrylocation;
|
||||
found := location.rank_address;
|
||||
END LOOP;
|
||||
|
||||
@ -2010,28 +2058,29 @@ BEGIN
|
||||
select name from country_name where country_code = searchcountrycode limit 1 INTO countryname;
|
||||
--RAISE WARNING '% % %',found,searchcountrycode,countryname;
|
||||
IF countryname IS NOT NULL THEN
|
||||
location := ROW(null, null, null, countryname, 'place', 'country', true, true, 4, 0)::addressline;
|
||||
location := ROW(null, null, null, countryname, 'place', 'country', null, true, true, 4, 0)::addressline;
|
||||
RETURN NEXT location;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
IF searchcountrycode IS NOT NULL THEN
|
||||
location := ROW(null, null, null, 'ref'=>searchcountrycode, 'place', 'country_code', true, false, 4, 0)::addressline;
|
||||
location := ROW(null, null, null, 'ref'=>searchcountrycode, 'place', 'country_code', null, true, false, 4, 0)::addressline;
|
||||
RETURN NEXT location;
|
||||
END IF;
|
||||
|
||||
IF searchhousename IS NOT NULL THEN
|
||||
location := ROW(in_place_id, null, null, searchhousename, 'place', 'house_name', true, true, 29, 0)::addressline;
|
||||
location := ROW(in_place_id, null, null, searchhousename, searchclass, searchtype, null, true, true, 29, 0)::addressline;
|
||||
-- location := ROW(in_place_id, null, null, searchhousename, 'place', 'house_name', null, true, true, 29, 0)::addressline;
|
||||
RETURN NEXT location;
|
||||
END IF;
|
||||
|
||||
IF searchhousenumber IS NOT NULL THEN
|
||||
location := ROW(in_place_id, null, null, 'ref'=>searchhousenumber, 'place', 'house_number', true, true, 28, 0)::addressline;
|
||||
location := ROW(in_place_id, null, null, 'ref'=>searchhousenumber, 'place', 'house_number', null, true, true, 28, 0)::addressline;
|
||||
RETURN NEXT location;
|
||||
END IF;
|
||||
|
||||
IF searchpostcode IS NOT NULL THEN
|
||||
location := ROW(null, null, null, 'ref'=>searchpostcode, 'place', 'postcode', true, true, 5, 0)::addressline;
|
||||
location := ROW(null, null, null, 'ref'=>searchpostcode, 'place', 'postcode', null, true, true, 5, 0)::addressline;
|
||||
RETURN NEXT location;
|
||||
END IF;
|
||||
|
||||
|
||||
@ -274,3 +274,18 @@ CREATE INDEX idx_placex_sector ON placex USING BTREE (geometry_sector,rank_addre
|
||||
|
||||
DROP SEQUENCE seq_postcodes;
|
||||
CREATE SEQUENCE seq_postcodes start 1;
|
||||
|
||||
drop table import_polygon_error;
|
||||
CREATE TABLE import_polygon_error (
|
||||
osm_type char(1),
|
||||
osm_id INTEGER,
|
||||
class TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
name HSTORE,
|
||||
country_code varchar(2),
|
||||
updated timestamp,
|
||||
errormessage text
|
||||
);
|
||||
SELECT AddGeometryColumn('import_polygon_error', 'prevgeometry', 4326, 'GEOMETRY', 2);
|
||||
SELECT AddGeometryColumn('import_polygon_error', 'newgeometry', 4326, 'GEOMETRY', 2);
|
||||
CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id);
|
||||
|
||||
@ -130,13 +130,9 @@
|
||||
{
|
||||
echo "Partitions\n";
|
||||
$bDidSomething = true;
|
||||
echo "here";
|
||||
$oDB =& getDB();
|
||||
echo "there";
|
||||
$sSQL = 'select partition from country_name order by country_code';
|
||||
var_dump($sSQL);
|
||||
$aPartitions = $oDB->getCol($sSQL);
|
||||
var_dump($aPartitions);
|
||||
if (PEAR::isError($aPartitions))
|
||||
{
|
||||
fail($aPartitions->getMessage());
|
||||
@ -147,11 +143,9 @@ var_dump($aPartitions);
|
||||
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
|
||||
foreach($aMatches as $aMatch)
|
||||
{
|
||||
var_dump($aMatch);
|
||||
$sResult = '';
|
||||
foreach($aPartitions as $sPartitionName)
|
||||
{
|
||||
var_dump($sPartitionName);
|
||||
$sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
|
||||
}
|
||||
$sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
|
||||
@ -371,8 +365,8 @@ var_dump($sPartitionName);
|
||||
{
|
||||
// Convert database DSN to psql paramaters
|
||||
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
|
||||
$sCMD = 'psql '.$aDSNInfo['database'];
|
||||
|
||||
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
|
||||
$sCMD = 'psql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database'];
|
||||
$aDescriptors = array(
|
||||
0 => array('pipe', 'r'),
|
||||
1 => array('pipe', 'w'),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user