From 3ea1b35b3243e4e2d061a207460b009883522d24 Mon Sep 17 00:00:00 2001 From: Brian Quinion Date: Mon, 7 Feb 2011 11:13:18 +0000 Subject: [PATCH] add logging of broken polygons, improve address export, better diff output in export, bug in address calculation(#3522), reaply #3520, lost in migration --- lib/lib.php | 2 +- nominatim/export.c | 236 +++++++++++++++++++++++++++++++-------------- nominatim/export.h | 18 +++- nominatim/import.c | 100 ++++++++++++++----- nominatim/index.c | 25 +++-- sql/functions.sql | 111 +++++++++++++++------ sql/tables.sql | 15 +++ utils/setup.php | 10 +- 8 files changed, 373 insertions(+), 144 deletions(-) diff --git a/lib/lib.php b/lib/lib.php index 9ee93105..115a9495 100644 --- a/lib/lib.php +++ b/lib/lib.php @@ -627,7 +627,7 @@ else { if (is_bool($xVal)) return $xVal?'true':'false'; - if (is_numeric($xVal)) return $xVal; +// if (is_numeric($xVal)) return $xVal; return '"'.str_replace('>','\\>',str_replace(array("\n","\r"),'\\n',str_replace(array("\n\r","\r\n"),'\\n',str_replace('"','\\"',$xVal)))).'"'; } } diff --git a/nominatim/export.c b/nominatim/export.c index 22565fff..8051c183 100644 --- a/nominatim/export.c +++ b/nominatim/export.c @@ -18,6 +18,8 @@ extern int verbose; +int mode = 0; + void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile) { xmlTextWriterPtr writer; @@ -129,7 +131,7 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch tuples = PQntuples(resPlaces); for (i = 0; i < tuples; i++) { - nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL); + nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL, NULL); rankTotalDone++; if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000); } @@ -150,7 +152,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn) pg_prepare_params[0] = PG_OID_INT8; res = PQprepare(conn, "placex_details", - "select osm_type, osm_id, class, type, name, housenumber, country_code, ST_AsText(geometry), admin_level, rank_address, rank_search from placex where place_id = $1", + "select placex.osm_type, placex.osm_id, placex.class, placex.type, placex.name, placex.housenumber, placex.country_code, ST_AsText(placex.geometry), placex.admin_level, placex.rank_address, placex.rank_search, placex.parent_place_id, parent.osm_type, parent.osm_id, placex.indexed_status from placex left outer join placex as parent on (placex.parent_place_id = parent.place_id) where placex.place_id = $1", 1, pg_prepare_params); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -224,23 +226,16 @@ xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile) fprintf(stderr, "xmlTextWriterWriteAttribute failed\n"); exit(EXIT_FAILURE); } - if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0) - { - fprintf(stderr, "xmlTextWriterStartElement failed\n"); - exit(EXIT_FAILURE); - } + + mode = 0; return writer; } void nominatim_exportXMLEnd(xmlTextWriterPtr writer) { - // End - if (xmlTextWriterEndElement(writer) < 0) - { - fprintf(stderr, "xmlTextWriterEndElement failed\n"); - exit(EXIT_FAILURE); - } + nominatim_exportEndMode(writer); + // End if (xmlTextWriterEndElement(writer) < 0) { @@ -255,135 +250,237 @@ void nominatim_exportXMLEnd(xmlTextWriterPtr writer) xmlFreeTextWriter(writer); } -/* - * Requirements: the prepared queries must exist - */ -void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex) +void nominatim_exportStartMode(xmlTextWriterPtr writer, int newMode) { - PGresult * res; - PGresult * resNames; - PGresult * resAddress; - PGresult * resExtraTags; + if (mode == newMode) return; - int i; + nominatim_exportEndMode(writer); + switch(newMode) + { + case 0: + break; + + case 1: + if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0) + { + fprintf(stderr, "xmlTextWriterStartElement failed\n"); + exit(EXIT_FAILURE); + } + break; + + case 2: + if (xmlTextWriterStartElement(writer, BAD_CAST "update") < 0) + { + fprintf(stderr, "xmlTextWriterStartElement failed\n"); + exit(EXIT_FAILURE); + } + break; + + case 3: + if (xmlTextWriterStartElement(writer, BAD_CAST "delete") < 0) + { + fprintf(stderr, "xmlTextWriterStartElement failed\n"); + exit(EXIT_FAILURE); + } + break; + } + mode = newMode; +} + +void nominatim_exportEndMode(xmlTextWriterPtr writer) +{ + if (!mode) return; + + if (xmlTextWriterEndElement(writer) < 0) + { + fprintf(stderr, "xmlTextWriterEndElement failed\n"); + exit(EXIT_FAILURE); + } +} + +void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet) +{ const char * paramValues[1]; int paramLengths[1]; int paramFormats[1]; uint64_t paramPlaceID; - paramPlaceID = PGint64(place_id); paramValues[0] = (char *)¶mPlaceID; paramLengths[0] = sizeof(paramPlaceID); paramFormats[0] = 1; - res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0); - if (PQresultStatus(res) != PGRES_TUPLES_OK) + querySet->res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0); + if (PQresultStatus(querySet->res) != PGRES_TUPLES_OK) { fprintf(stderr, "placex_details: SELECT failed: %s", PQerrorMessage(conn)); - PQclear(res); + PQclear(querySet->res); exit(EXIT_FAILURE); } - resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0); - if (PQresultStatus(resNames) != PGRES_TUPLES_OK) + querySet->resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0); + if (PQresultStatus(querySet->resNames) != PGRES_TUPLES_OK) { fprintf(stderr, "placex_names: SELECT failed: %s", PQerrorMessage(conn)); - PQclear(resNames); + PQclear(querySet->resNames); exit(EXIT_FAILURE); } - resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0); - if (PQresultStatus(resAddress) != PGRES_TUPLES_OK) + querySet->resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0); + if (PQresultStatus(querySet->resAddress) != PGRES_TUPLES_OK) { fprintf(stderr, "placex_address: SELECT failed: %s", PQerrorMessage(conn)); - PQclear(resAddress); + PQclear(querySet->resAddress); exit(EXIT_FAILURE); } - resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0); - if (PQresultStatus(resExtraTags) != PGRES_TUPLES_OK) + querySet->resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0); + if (PQresultStatus(querySet->resExtraTags) != PGRES_TUPLES_OK) { fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn)); - PQclear(resExtraTags); + PQclear(querySet->resExtraTags); exit(EXIT_FAILURE); } +} - if (writer_mutex) pthread_mutex_lock( writer_mutex ); +void nominatim_exportFreeQueries(struct export_data * querySet) +{ + PQclear(querySet->res); + PQclear(querySet->resNames); + PQclear(querySet->resAddress); + PQclear(querySet->resExtraTags); +} + +/* + * Requirements: the prepared queries must exist + */ +void nominatim_exportPlace(uint64_t place_id, PGconn * conn, + xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet) +{ + struct export_data querySet; + + int i; + + nominatim_exportPlaceQueries(place_id, conn, &querySet); + + // Add, modify or delete? + if (prevQuerySet) + { + if ((PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "100") == 0) || PQntuples(querySet.res)) + { + // Delete + if (writer_mutex) pthread_mutex_lock( writer_mutex ); + nominatim_exportStartMode(writer, 3); + xmlTextWriterStartElement(writer, BAD_CAST "feature"); + xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id); + xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 0)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 1)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 2)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 3)); + xmlTextWriterEndElement(writer); + if (writer_mutex) pthread_mutex_unlock( writer_mutex ); + nominatim_exportFreeQueries(&querySet); + return; + } + if (PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "1") == 0) + { + // Add + if (writer_mutex) pthread_mutex_lock( writer_mutex ); + nominatim_exportStartMode(writer, 1); + } + else + { + // Update, but only if something has changed + + // TODO: detect changes + + if (writer_mutex) pthread_mutex_lock( writer_mutex ); + nominatim_exportStartMode(writer, 2); + } + } + else + { + // Add + if (writer_mutex) pthread_mutex_lock( writer_mutex ); + nominatim_exportStartMode(writer, 1); + } xmlTextWriterStartElement(writer, BAD_CAST "feature"); xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id); - xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(res, 0, 0)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(res, 0, 1)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(res, 0, 2)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(res, 0, 3)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(res, 0, 9)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(res, 0, 10)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.res, 0, 0)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.res, 0, 1)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.res, 0, 2)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.res, 0, 3)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.res, 0, 9)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(querySet.res, 0, 10)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_place_id", BAD_CAST PQgetvalue(querySet.res, 0, 11)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_type", BAD_CAST PQgetvalue(querySet.res, 0, 12)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_id", BAD_CAST PQgetvalue(querySet.res, 0, 13)); - if (PQntuples(resNames)) + if (PQntuples(querySet.resNames)) { xmlTextWriterStartElement(writer, BAD_CAST "names"); - for (i = 0; i < PQntuples(resNames); i++) + for (i = 0; i < PQntuples(querySet.resNames); i++) { xmlTextWriterStartElement(writer, BAD_CAST "name"); - xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resNames, i, 0)); - xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resNames, i, 1)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resNames, i, 0)); + xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resNames, i, 1)); xmlTextWriterEndElement(writer); } xmlTextWriterEndElement(writer); } - if (PQgetvalue(res, 0, 5) && strlen(PQgetvalue(res, 0, 5))) + if (PQgetvalue(querySet.res, 0, 5) && strlen(PQgetvalue(querySet.res, 0, 5))) { xmlTextWriterStartElement(writer, BAD_CAST "houseNumber"); - xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 5)); + xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 5)); xmlTextWriterEndElement(writer); } - if (PQgetvalue(res, 0, 8) && strlen(PQgetvalue(res, 0, 8))) + if (PQgetvalue(querySet.res, 0, 8) && strlen(PQgetvalue(querySet.res, 0, 8))) { xmlTextWriterStartElement(writer, BAD_CAST "adminLevel"); - xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 8)); + xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 8)); xmlTextWriterEndElement(writer); } - if (PQgetvalue(res, 0, 6) && strlen(PQgetvalue(res, 0, 6))) + if (PQgetvalue(querySet.res, 0, 6) && strlen(PQgetvalue(querySet.res, 0, 6))) { xmlTextWriterStartElement(writer, BAD_CAST "countryCode"); - xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 6)); + xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 6)); xmlTextWriterEndElement(writer); } - if (PQntuples(resAddress) > 0) + if (PQntuples(querySet.resAddress) > 0) { xmlTextWriterStartElement(writer, BAD_CAST "address"); - for (i = 0; i < PQntuples(resAddress); i++) + for (i = 0; i < PQntuples(querySet.resAddress); i++) { - xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(resAddress, i, 5)))); - xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(resAddress, i, 5)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resAddress, i, 0)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(resAddress, i, 1)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(resAddress, i, 2)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(resAddress, i, 3)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(resAddress, i, 4)); - xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(resAddress, i, 6)); + xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(querySet.resAddress, i, 5)))); + xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.resAddress, i, 5)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resAddress, i, 0)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.resAddress, i, 1)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.resAddress, i, 2)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.resAddress, i, 3)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(querySet.resAddress, i, 4)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(querySet.resAddress, i, 6)); xmlTextWriterEndElement(writer); } xmlTextWriterEndElement(writer); } - if (PQntuples(resExtraTags)) + if (PQntuples(querySet.resExtraTags)) { xmlTextWriterStartElement(writer, BAD_CAST "tags"); - for (i = 0; i < PQntuples(resExtraTags); i++) + for (i = 0; i < PQntuples(querySet.resExtraTags); i++) { xmlTextWriterStartElement(writer, BAD_CAST "tag"); - xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resExtraTags, i, 0)); - xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resExtraTags, i, 1)); + xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resExtraTags, i, 0)); + xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resExtraTags, i, 1)); xmlTextWriterEndElement(writer); } @@ -392,17 +489,14 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry"); - xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 7)); + xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 7)); xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer); // if (writer_mutex) pthread_mutex_unlock( writer_mutex ); - PQclear(res); - PQclear(resNames); - PQclear(resAddress); - PQclear(resExtraTags); + nominatim_exportFreeQueries(&querySet); } const char * getRankLabel(int rank) diff --git a/nominatim/export.h b/nominatim/export.h index 7533db2e..c7a302c4 100644 --- a/nominatim/export.h +++ b/nominatim/export.h @@ -5,11 +5,27 @@ #include #include +struct export_data +{ + PGresult * res; + PGresult * resNames; + PGresult * resAddress; + PGresult * resExtraTags; +}; + void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile); void nominatim_exportCreatePreparedQueries(PGconn * conn); + xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile); void nominatim_exportXMLEnd(xmlTextWriterPtr writer); -void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex); + +void nominatim_exportEndMode(xmlTextWriterPtr writer); + +void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet); +void nominatim_exportFreeQueries(struct export_data * querySet); + +void nominatim_exportPlace(uint64_t place_id, PGconn * conn, + xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet); const char * getRankLabel(int rank); #endif diff --git a/nominatim/import.c b/nominatim/import.c index 03cf6fa3..89851d2f 100644 --- a/nominatim/import.c +++ b/nominatim/import.c @@ -50,6 +50,9 @@ struct feature xmlChar * rankAddress; xmlChar * rankSearch; xmlChar * countryCode; + xmlChar * parentPlaceID; + xmlChar * parentType; + xmlChar * parentID; xmlChar * adminLevel; xmlChar * houseNumber; xmlChar * geometry; @@ -67,6 +70,7 @@ int featureNameLines = 0; int featureExtraTagLines = 0; int featureCount = 0; xmlHashTablePtr partionTableTagsHash; +xmlHashTablePtr partionTableTagsHashDelete; char featureNameString[MAX_FEATURENAMESTRING]; char featureExtraTagString[MAX_FEATUREEXTRATAGSTRING]; @@ -135,13 +139,17 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name) feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank"); feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance"); + feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id"); + feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type"); + feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id"); + feature.countryCode = NULL; feature.adminLevel = NULL; feature.houseNumber = NULL; feature.geometry = NULL; featureAddressLines = 0; featureNameLines = 0; - featureExtraTagLines = 0; + featureExtraTagLines = 0; return; } @@ -312,7 +320,6 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name) void EndElement(xmlTextReaderPtr reader, const xmlChar *name) { PGresult * res; - PGresult * resPlaceID; const char * paramValues[11]; char * place_id; char * partionQueryName; @@ -348,9 +355,9 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) } } */ - place_id = feature.placeID; + place_id = (char *)feature.placeID; - if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE) + if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD) { paramValues[0] = (const char *)place_id; res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0); @@ -379,6 +386,19 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) exit(EXIT_FAILURE); } PQclear(res); + + partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value); + if (partionQueryName) + { + res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn)); + PQclear(res); + exit(EXIT_FAILURE); + } + PQclear(res); + } } if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD) @@ -398,8 +418,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) lineValueLen = 0; for (i = 0; i < featureNameLines; i++) { - lineTypeLen = strlen(BAD_CAST featureName[i].type); - lineValueLen = strlen(BAD_CAST featureName[i].value); + lineTypeLen = (int)strlen((char *) featureName[i].type); + lineValueLen = (int)strlen((char *) featureName[i].value); if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING) { fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value); @@ -407,11 +427,11 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) } if (namePos) strcpy(featureNameString+(namePos++), ","); strcpy(featureNameString+(namePos++), "\""); - strcpy(featureNameString+namePos, BAD_CAST featureName[i].type); + strcpy(featureNameString+namePos, (char*) featureName[i].type); namePos += lineTypeLen; strcpy(featureNameString+namePos, "\"=>\""); namePos += 4; - strcpy(featureNameString+namePos, BAD_CAST featureName[i].value); + strcpy(featureNameString+namePos, (char *) featureName[i].value); namePos += lineValueLen; strcpy(featureNameString+(namePos++), "\""); } @@ -426,8 +446,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) lineValueLen = 0; for (i = 0; i < featureExtraTagLines; i++) { - lineTypeLen = strlen(BAD_CAST featureExtraTag[i].type); - lineValueLen = strlen(BAD_CAST featureExtraTag[i].value); + lineTypeLen = strlen((char *) featureExtraTag[i].type); + lineValueLen = strlen((char *) featureExtraTag[i].value); if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING) { fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value); @@ -435,30 +455,36 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) } if (namePos) strcpy(featureExtraTagString+(namePos++),","); strcpy(featureExtraTagString+(namePos++), "\""); - strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].type); + strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type); namePos += lineTypeLen; strcpy(featureExtraTagString+namePos, "\"=>\""); namePos += 4; - strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].value); + strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value); namePos += lineValueLen; strcpy(featureExtraTagString+(namePos++), "\""); } } paramValues[6] = (const char *)featureExtraTagString; - paramValues[7] = (const char *)feature.adminLevel; - paramValues[8] = (const char *)feature.houseNumber; - paramValues[9] = (const char *)feature.rankAddress; - paramValues[10] = (const char *)feature.rankSearch; - paramValues[11] = (const char *)feature.geometry; - res = PQexecPrepared(conn, "placex_insert", 12, paramValues, NULL, NULL, 0); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + paramValues[7] = (const char *)feature.parentPlaceID; + + paramValues[8] = (const char *)feature.adminLevel; + paramValues[9] = (const char *)feature.houseNumber; + paramValues[10] = (const char *)feature.rankAddress; + paramValues[11] = (const char *)feature.rankSearch; + paramValues[12] = (const char *)feature.geometry; + if (strlen(paramValues[3])) { - fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn)); - PQclear(res); - exit(EXIT_FAILURE); + res = PQexecPrepared(conn, "placex_insert", 13, paramValues, NULL, NULL, 0); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn)); + fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]); + PQclear(res); + exit(EXIT_FAILURE); + } + PQclear(res); } - PQclear(res); for (i = 0; i < featureAddressLines; i++) { @@ -513,7 +539,6 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name) exit(EXIT_FAILURE); } PQclear(res); - } } @@ -595,6 +620,7 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons } partionTableTagsHash = xmlHashCreate(200); + partionTableTagsHashDelete = xmlHashCreate(200); partionTagsFile = fopen(partionTagsFilename, "rt"); if (!partionTagsFile) @@ -636,6 +662,27 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons } xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName); + + partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2); + strcpy(partionQueryName, "partition_delete_"); + strcat(partionQueryName, osmkey); + strcat(partionQueryName, "_"); + strcat(partionQueryName, osmvalue); + + strcpy(partionQuerySQL, "delete from place_classtype_"); + strcat(partionQuerySQL, osmkey); + strcat(partionQuerySQL, "_"); + strcat(partionQuerySQL, osmvalue); + strcat(partionQuerySQL, " where place_id = $1::integer"); + + res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn)); + exit(EXIT_FAILURE); + } + + xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName); } res = PQprepare(conn, "get_new_place_id", @@ -657,8 +704,8 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons } res = PQprepare(conn, "placex_insert", - "insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,admin_level,housenumber,rank_address,rank_search,geometry) " - "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, st_setsrid($12, 4326))", + "insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) " + "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, st_setsrid($13, 4326))", 12, NULL); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -740,6 +787,7 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons xmlFreeTextReader(reader); xmlHashFree(partionTableTagsHash, NULL); + xmlHashFree(partionTableTagsHashDelete, NULL); return 0; } diff --git a/nominatim/index.c b/nominatim/index.c index bee8b3da..5d10994c 100644 --- a/nominatim/index.c +++ b/nominatim/index.c @@ -40,6 +40,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co int i; int iSector; int iResult; + int bSkip; const char *paramValues[2]; int paramLengths[2]; @@ -160,7 +161,8 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co // if (rank < 16) // resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1); // else - resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); + resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); + if (PQresultStatus(resSectors) != PGRES_TUPLES_OK) { fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn)); @@ -215,7 +217,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co if (iSector < PQntuples(resSectors)) { sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); - //printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); +// printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); // Get all the place_id's for this sector paramRank = PGint32(rank); @@ -226,9 +228,9 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co paramValues[1] = (char *)¶mSector; paramLengths[1] = sizeof(paramSector); paramFormats[1] = 1; - if (rankTotalTuples-rankCountTuples < num_threads*20) + if (rankTotalTuples-rankCountTuples < num_threads*1000) { - iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); + iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); } else { @@ -292,7 +294,10 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co PQclear(resPlaces); } - if (rankTotalTuples-rankCountTuples < num_threads*20) iSector = PQntuples(resSectors); + if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors)) + { + iSector = PQntuples(resSectors) - 1; + } } // Finished rank printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); @@ -309,6 +314,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co void *nominatim_indexThread(void * thread_data_in) { struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in; + struct export_data querySet; PGresult *res; @@ -339,6 +345,12 @@ void *nominatim_indexThread(void * thread_data_in) int done = 0; while(!done) { + if (thread_data->writer) + { + nominatim_exportPlaceQueries(place_id, thread_data->conn, &querySet); + } + + paramPlaceID = PGint32(place_id); paramValues[0] = (char *)¶mPlaceID; paramLengths[0] = sizeof(paramPlaceID); @@ -366,7 +378,8 @@ void *nominatim_indexThread(void * thread_data_in) if (thread_data->writer) { - nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex); + nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex, &querySet); + nominatim_exportFreeQueries(&querySet); } } diff --git a/sql/functions.sql b/sql/functions.sql index 9b089ad8..157c1525 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -1102,7 +1102,7 @@ BEGIN -- RETURN NULL; -- END IF; --- RETURN NEW; -- The following is not needed until doing diff updates, and slows the main index process down + RETURN NEW; -- The following is not needed until doing diff updates, and slows the main index process down IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN -- Performance: We just can't handle re-indexing for country level changes @@ -1218,6 +1218,12 @@ BEGIN result := deleteSearchName(NEW.partition, NEW.place_id); DELETE FROM place_addressline WHERE place_id = NEW.place_id; DELETE FROM place_boundingbox where place_id = NEW.place_id; + result := deleteRoad(NEW.partition, NEW.place_id); + result := deleteLocationArea(NEW.partition, NEW.place_id); + + NEW.country_code := lower(get_country_code(NEW.geometry, NEW.country_code)); + NEW.partition := get_partition(NEW.geometry, NEW.country_code); + NEW.geometry_sector := geometry_sector(NEW.partition, NEW.geometry); -- Adding ourselves to the list simplifies address calculations later INSERT INTO place_addressline VALUES (NEW.place_id, NEW.place_id, true, true, 0, NEW.rank_address); @@ -1345,7 +1351,7 @@ BEGIN END LOOP; END IF; ---RAISE WARNING 'x3'; +--RAISE WARNING 'x3 %',NEW.parent_place_id; IF NEW.parent_place_id IS NULL AND NEW.street IS NOT NULL THEN address_street_word_id := get_name_id(make_standard_name(NEW.street)); @@ -1356,11 +1362,13 @@ BEGIN END IF; END IF; ---RAISE WARNING 'x4'; +--RAISE WARNING 'x4 %',NEW.parent_place_id; -- Still nothing, just use the nearest road - FOR location IN SELECT place_id FROM getNearestRoadFeature(NEW.partition, place_centroid) LOOP - NEW.parent_place_id := location.place_id; - END LOOP; + IF NEW.parent_place_id IS NULL THEN + FOR location IN SELECT place_id FROM getNearestRoadFeature(NEW.partition, place_centroid) LOOP + NEW.parent_place_id := location.place_id; + END LOOP; + END IF; --return NEW; --RAISE WARNING 'x6 %',NEW.parent_place_id; @@ -1548,6 +1556,10 @@ BEGIN DELETE FROM place_addressline where address_place_id = OLD.place_id; + b := deleteRoad(OLD.partition, OLD.place_id); + + update placex set indexed_status = 2 where parent_place_id = OLD.place_id and indexed_status = 0; + END IF; IF OLD.rank_address < 26 THEN @@ -1596,7 +1608,7 @@ DECLARE partition INTEGER; BEGIN - IF FALSE AND NEW.osm_type = 'R' THEN + IF FALSE and NEW.osm_type = 'R' THEN RAISE WARNING '-----------------------------------------------------------------------------------'; RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry); select * from placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type INTO existingplacex; @@ -1612,6 +1624,8 @@ BEGIN END IF; IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN + INSERT INTO import_polygon_error values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.country_code, + now(), ST_IsValidReason(NEW.geometry), null, NEW.geometry); -- RAISE WARNING 'Invalid Geometry: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type; RETURN null; END IF; @@ -1629,13 +1643,26 @@ BEGIN -- Handle a place changing type by removing the old data -- My generated 'place' types are causing havok because they overlap with real tags - -- TODO: move them to their own special purpose tag to avoid collisions - IF existing.osm_type IS NULL AND (NEW.type not in ('postcode','house','houses')) THEN - DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type not in ('postcode','house','houses'); - END IF; + -- TODO: move them to their own special purpose key/class to avoid collisions +-- IF existing.osm_type IS NULL AND (NEW.type not in ('postcode','house','houses')) THEN +-- DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type not in ('postcode','house','houses'); +-- END IF; -- RAISE WARNING 'Existing: %',existing.place_id; + -- Log and discard + IF existing.geometry is not null AND st_isvalid(existing.geometry) + AND st_area(existing.geometry) > 0.02 + AND ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') + AND st_area(NEW.geometry) < st_area(existing.geometry)*0.5 + THEN + INSERT INTO import_polygon_error values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.country_code, now(), + 'Area reduced from '||st_area(existing.geometry)||' to '||st_area(NEW.geometry), existing.geometry, NEW.geometry); + RETURN null; + END IF; + + DELETE from import_polygon_error where osm_type = NEW.osm_type and osm_id = NEW.osm_id; + -- To paraphrase, if there isn't an existing item, OR if the admin level has changed, OR if it is a major change in geometry IF existing.osm_type IS NULL OR existingplacex.osm_type IS NULL @@ -1652,7 +1679,6 @@ BEGIN -- RAISE WARNING 'no existing placex %', existingplacex; -- END IF; - -- RAISE WARNING 'delete and replace'; IF existing.osm_type IS NOT NULL THEN @@ -1719,7 +1745,7 @@ BEGIN select geometry from placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type into existinggeometry; -- Performance limit - IF st_area(NEW.geometry) < 1 AND st_area(existinggeometry) < 1 THEN + IF st_area(NEW.geometry) < 0.000000001 AND st_area(existinggeometry) < 1 THEN -- re-index points that have moved in / out of the polygon, could be done as a single query but postgres gets the index usage wrong update placex set indexed_status = 2 where indexed_status = 0 and @@ -1767,8 +1793,8 @@ BEGIN -- performance, can't take the load of re-indexing a whole country / huge area IF st_area(NEW.geometry) < 0.5 THEN - UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = existingplacex.place_id - and placex.place_id = place_addressline.place_id and indexed_status = 0; +-- UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = existingplacex.place_id +-- and placex.place_id = place_addressline.place_id and indexed_status = 0; END IF; END IF; @@ -1809,7 +1835,7 @@ BEGIN geometry = NEW.geometry where place_id = existingplacex.place_id; --- now done as part of indexing +-- now done as part of insert -- partition := get_partition(NEW.geometry, existingplacex.country_code); -- result := update_location(partition, existingplacex.place_id, existingplacex.country_code, NEW.name, existingplacex.rank_search, existingplacex.rank_address, NEW.geometry); @@ -1939,6 +1965,7 @@ create type addressline as ( name HSTORE, class TEXT, type TEXT, + admin_level INTEGER, fromarea BOOLEAN, isaddress BOOLEAN, rank_address INTEGER, @@ -1953,28 +1980,32 @@ DECLARE search TEXT[]; found INTEGER; location RECORD; + countrylocation RECORD; searchcountrycode varchar(2); searchhousenumber TEXT; searchhousename HSTORE; searchrankaddress INTEGER; searchpostcode TEXT; + searchclass TEXT; + searchtype TEXT; countryname HSTORE; + hadcountry BOOLEAN; BEGIN - select parent_place_id,'us', housenumber, 30, postcode, null from location_property_tiger + select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_tiger WHERE place_id = in_place_id - INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename; + INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype; IF for_place_id IS NULL THEN - select parent_place_id,'us', housenumber, 30, postcode, null from location_property_aux + select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux WHERE place_id = in_place_id - INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename; + INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype; END IF; IF for_place_id IS NULL THEN - select parent_place_id, country_code, housenumber, rank_address, postcode, name from placex + select parent_place_id, country_code, housenumber, rank_address, postcode, name, class, type from placex WHERE place_id = in_place_id and rank_address = 30 - INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename; + INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype; END IF; IF for_place_id IS NULL THEN @@ -1986,12 +2017,13 @@ BEGIN --RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode; found := 1000; + hadcountry := false; FOR location IN - select placex.place_id, osm_type, osm_id, + select placex.place_id, osm_type, osm_id, CASE WHEN class = 'place' and type = 'postcode' THEN 'name' => postcode ELSE name END as name, - class, type, fromarea, isaddress, + class, type, admin_level, fromarea, isaddress, CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address, - distance + distance,country_code from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = for_place_id and ((cached_rank_address > 0 AND cached_rank_address < searchrankaddress) OR address_place_id = for_place_id) @@ -1999,10 +2031,26 @@ BEGIN order by rank_address desc,isaddress desc,fromarea desc,distance asc,rank_search desc LOOP --RAISE WARNING '%',location; + IF searchcountrycode IS NULL AND location.country_code IS NOT NULL THEN + searchcountrycode := location.country_code; + END IF; IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN location.isaddress := FALSE; END IF; - RETURN NEXT location; + IF location.rank_address = 4 AND location.isaddress THEN + hadcountry := true; + END IF; + IF location.rank_address < 4 AND NOT hadcountry THEN + select name from country_name where country_code = searchcountrycode limit 1 INTO countryname; + IF countryname IS NOT NULL THEN + countrylocation := ROW(null, null, null, countryname, 'place', 'country', null, true, true, 4, 0)::addressline; + RETURN NEXT countrylocation; + END IF; + END IF; + countrylocation := ROW(location.place_id, location.osm_type, location.osm_id, location.name, location.class, + location.type, location.admin_level, location.fromarea, location.isaddress, location.rank_address, + location.distance)::addressline; + RETURN NEXT countrylocation; found := location.rank_address; END LOOP; @@ -2010,28 +2058,29 @@ BEGIN select name from country_name where country_code = searchcountrycode limit 1 INTO countryname; --RAISE WARNING '% % %',found,searchcountrycode,countryname; IF countryname IS NOT NULL THEN - location := ROW(null, null, null, countryname, 'place', 'country', true, true, 4, 0)::addressline; + location := ROW(null, null, null, countryname, 'place', 'country', null, true, true, 4, 0)::addressline; RETURN NEXT location; END IF; END IF; IF searchcountrycode IS NOT NULL THEN - location := ROW(null, null, null, 'ref'=>searchcountrycode, 'place', 'country_code', true, false, 4, 0)::addressline; + location := ROW(null, null, null, 'ref'=>searchcountrycode, 'place', 'country_code', null, true, false, 4, 0)::addressline; RETURN NEXT location; END IF; IF searchhousename IS NOT NULL THEN - location := ROW(in_place_id, null, null, searchhousename, 'place', 'house_name', true, true, 29, 0)::addressline; + location := ROW(in_place_id, null, null, searchhousename, searchclass, searchtype, null, true, true, 29, 0)::addressline; +-- location := ROW(in_place_id, null, null, searchhousename, 'place', 'house_name', null, true, true, 29, 0)::addressline; RETURN NEXT location; END IF; IF searchhousenumber IS NOT NULL THEN - location := ROW(in_place_id, null, null, 'ref'=>searchhousenumber, 'place', 'house_number', true, true, 28, 0)::addressline; + location := ROW(in_place_id, null, null, 'ref'=>searchhousenumber, 'place', 'house_number', null, true, true, 28, 0)::addressline; RETURN NEXT location; END IF; IF searchpostcode IS NOT NULL THEN - location := ROW(null, null, null, 'ref'=>searchpostcode, 'place', 'postcode', true, true, 5, 0)::addressline; + location := ROW(null, null, null, 'ref'=>searchpostcode, 'place', 'postcode', null, true, true, 5, 0)::addressline; RETURN NEXT location; END IF; diff --git a/sql/tables.sql b/sql/tables.sql index b8c63f71..f3a82b3b 100644 --- a/sql/tables.sql +++ b/sql/tables.sql @@ -274,3 +274,18 @@ CREATE INDEX idx_placex_sector ON placex USING BTREE (geometry_sector,rank_addre DROP SEQUENCE seq_postcodes; CREATE SEQUENCE seq_postcodes start 1; + +drop table import_polygon_error; +CREATE TABLE import_polygon_error ( + osm_type char(1), + osm_id INTEGER, + class TEXT NOT NULL, + type TEXT NOT NULL, + name HSTORE, + country_code varchar(2), + updated timestamp, + errormessage text + ); +SELECT AddGeometryColumn('import_polygon_error', 'prevgeometry', 4326, 'GEOMETRY', 2); +SELECT AddGeometryColumn('import_polygon_error', 'newgeometry', 4326, 'GEOMETRY', 2); +CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id); diff --git a/utils/setup.php b/utils/setup.php index 26d9e8c6..352aa943 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -130,13 +130,9 @@ { echo "Partitions\n"; $bDidSomething = true; -echo "here"; $oDB =& getDB(); -echo "there"; $sSQL = 'select partition from country_name order by country_code'; -var_dump($sSQL); $aPartitions = $oDB->getCol($sSQL); -var_dump($aPartitions); if (PEAR::isError($aPartitions)) { fail($aPartitions->getMessage()); @@ -147,11 +143,9 @@ var_dump($aPartitions); preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); foreach($aMatches as $aMatch) { -var_dump($aMatch); $sResult = ''; foreach($aPartitions as $sPartitionName) { -var_dump($sPartitionName); $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); } $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); @@ -371,8 +365,8 @@ var_dump($sPartitionName); { // Convert database DSN to psql paramaters $aDSNInfo = DB::parseDSN(CONST_Database_DSN); - $sCMD = 'psql '.$aDSNInfo['database']; - + if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; + $sCMD = 'psql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']; $aDescriptors = array( 0 => array('pipe', 'r'), 1 => array('pipe', 'w'),