osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-11-29 08:36:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

aux properties, memory leaks, tweaks to search order

Browse Source
This commit is contained in:
Brian Quinion 2011-01-21 10:40:44 +00:00
parent 2be00f37c1
commit 31273a42bd
11 changed files with 401 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
nominatim/export.c
View File

@ -161,7 +161,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
pg_prepare_params[0] = PG_OID_INT8; pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_address", res = PQprepare(conn, "placex_address",
"select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc", "select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc,osm_type,osm_id",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
@ -172,7 +172,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
pg_prepare_params[0] = PG_OID_INT8; pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_names", res = PQprepare(conn, "placex_names",
"select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x", "select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x order by (each(name)).key",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
@ -183,7 +183,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
pg_prepare_params[0] = PG_OID_INT8; pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_extratags", res = PQprepare(conn, "placex_extratags",
"select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x", "select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x order by (each(extratags)).key",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
@ -402,6 +402,7 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
PQclear(res); PQclear(res);
PQclear(resNames); PQclear(resNames);
PQclear(resAddress); PQclear(resAddress);
PQclear(resExtraTags);
} }
const char * getRankLabel(int rank) const char * getRankLabel(int rank)

111
nominatim/import.c
View File

@ -16,11 +16,11 @@
typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t; typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t; typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
#define MAX_FEATUREADDRESS 500 #define MAX_FEATUREADDRESS 5000
#define MAX_FEATURENAMES 1000 #define MAX_FEATURENAMES 10000
#define MAX_FEATUREEXTRATAGS 100 #define MAX_FEATUREEXTRATAGS 10000
#define MAX_FEATURENAMESTRING 100000 #define MAX_FEATURENAMESTRING 1000000
#define MAX_FEATUREEXTRATAGSTRING 50000 #define MAX_FEATUREEXTRATAGSTRING 500000
struct feature_address struct feature_address
{ {
@ -42,7 +42,7 @@ struct feature_tag
struct feature struct feature
{ {
int placeID; xmlChar * placeID;
xmlChar * type; xmlChar * type;
xmlChar * id; xmlChar * id;
xmlChar * key; xmlChar * key;
@ -92,7 +92,7 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
} }
else else
{ {
fprintf( stderr, "Unknown osmStructured version %f\n", version ); fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
exit_nicely(); exit_nicely();
} }
} }
@ -127,6 +127,7 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
if (xmlStrEqual(name, BAD_CAST "feature")) if (xmlStrEqual(name, BAD_CAST "feature"))
{ {
feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type"); feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key"); feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
@ -140,32 +141,39 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
feature.geometry = NULL; feature.geometry = NULL;
featureAddressLines = 0; featureAddressLines = 0;
featureNameLines = 0; featureNameLines = 0;
featureExtraTagLines = 0;
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "names")) return; if (xmlStrEqual(name, BAD_CAST "names")) return;
if (xmlStrEqual(name, BAD_CAST "name")) if (xmlStrEqual(name, BAD_CAST "name"))
{ {
featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type"); if (featureNameLines < MAX_FEATURENAMES)
featureName[featureNameLines].value = xmlTextReaderReadString(reader);
featureNameLines++;
if (featureNameLines >= MAX_FEATURENAMES)
{ {
fprintf( stderr, "Too many name elements\n"); featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
exit_nicely(); featureName[featureNameLines].value = xmlTextReaderReadString(reader);
featureNameLines++;
}
else
{
fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
// exit_nicely();
} }
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "tags")) return; if (xmlStrEqual(name, BAD_CAST "tags")) return;
if (xmlStrEqual(name, BAD_CAST "tag")) if (xmlStrEqual(name, BAD_CAST "tag"))
{ {
featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type"); if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader); {
featureExtraTagLines++; featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
if (featureExtraTagLines >= MAX_FEATUREEXTRATAGS) featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
featureExtraTagLines++;
}
else
{ {
fprintf( stderr, "Too many extra tag elements\n"); fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
exit_nicely(); // exit_nicely();
} }
return; return;
} }
@ -261,36 +269,39 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
} }
if (isAddressLine) if (isAddressLine)
{ {
value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank"); if (featureAddressLines < MAX_FEATUREADDRESS)
if (!value) {
{ value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
fprintf( stderr, "Address element missing rank\n"); if (!value)
exit_nicely(); {
} fprintf( stderr, "Address element missing rank\n");
featureAddress[featureAddressLines].rankAddress = atoi(value); exit_nicely();
xmlFree(value); }
featureAddress[featureAddressLines].rankAddress = atoi(value);
xmlFree(value);
value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress"); value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
if (!value) if (!value)
{ {
fprintf( stderr, "Address element missing rank\n"); fprintf( stderr, "Address element missing rank\n");
exit_nicely(); exit_nicely();
} }
if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t"); if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
else strcpy(featureAddress[featureAddressLines].isAddress, "f"); else strcpy(featureAddress[featureAddressLines].isAddress, "f");
xmlFree(value); xmlFree(value);
featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type"); featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key"); featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value"); featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance"); featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
featureAddressLines++; featureAddressLines++;
if (featureAddressLines >= MAX_FEATUREADDRESS) }
else
{ {
fprintf( stderr, "Too many address elements\n"); fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
exit_nicely(); // exit_nicely();
} }
return; return;
@ -311,7 +322,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
{ {
featureCount++; featureCount++;
if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000); if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
/*
if (fileMode == FILEMODE_ADD) if (fileMode == FILEMODE_ADD)
{ {
resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0); resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
@ -336,7 +347,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
place_id = PQgetvalue(resPlaceID, 0, 0); */
place_id = feature.placeID;
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE) if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE)
{ {
@ -517,6 +529,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
} }
} }
xmlFree(feature.placeID);
xmlFree(feature.type); xmlFree(feature.type);
xmlFree(feature.id); xmlFree(feature.id);
xmlFree(feature.key); xmlFree(feature.key);
@ -529,7 +542,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
if (feature.houseNumber) xmlFree(feature.houseNumber); if (feature.houseNumber) xmlFree(feature.houseNumber);
if (feature.geometry) xmlFree(feature.geometry); if (feature.geometry) xmlFree(feature.geometry);
PQclear(resPlaceID); // PQclear(resPlaceID);
} }
} }

49
nominatim/index.c
View File

@ -157,9 +157,9 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
paramValues[0] = (char *)&paramRank; paramValues[0] = (char *)&paramRank;
paramLengths[0] = sizeof(paramRank); paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1; paramFormats[0] = 1;
if (rank < 16) // if (rank < 16)
resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1); // resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
else // else
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK) if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
{ {
@ -226,10 +226,14 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
paramValues[1] = (char *)&paramSector; paramValues[1] = (char *)&paramSector;
paramLengths[1] = sizeof(paramSector); paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1; paramFormats[1] = 1;
if (rank < 16) if (rankTotalTuples-rankCountTuples < num_threads*20)
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); {
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
}
else else
{
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1); iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
}
if (!iResult) if (!iResult)
{ {
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
@ -288,6 +292,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
PQclear(resPlaces); PQclear(resPlaces);
} }
if (rankTotalTuples-rankCountTuples < num_threads*20) iSector = PQntuples(resSectors);
} }
// Finished rank // Finished rank
printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
@ -331,16 +336,30 @@ void *nominatim_indexThread(void * thread_data_in)
if (verbose) printf(" Processing place_id %d\n", place_id); if (verbose) printf(" Processing place_id %d\n", place_id);
updateStartTime = time(0); updateStartTime = time(0);
paramPlaceID = PGint32(place_id); int done = 0;
paramValues[0] = (char *)&paramPlaceID; while(!done)
paramLengths[0] = sizeof(paramPlaceID); {
paramFormats[0] = 1; paramPlaceID = PGint32(place_id);
res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1); paramValues[0] = (char *)&paramPlaceID;
if (PQresultStatus(res) != PGRES_COMMAND_OK) paramLengths[0] = sizeof(paramPlaceID);
{ paramFormats[0] = 1;
fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn)); res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1);
PQclear(res); if (PQresultStatus(res) == PGRES_COMMAND_OK)
exit(EXIT_FAILURE); done = 1;
else
{
if (strncmp(PQerrorMessage(thread_data->conn), "ERROR: deadlock detected", 25))
{
fprintf(stderr, "index_placex: UPDATE failed - deadlock, retrying\n");
}
else
{
fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
PQclear(res);
sleep(5);
// exit(EXIT_FAILURE);
}
}
} }
PQclear(res); PQclear(res);
if (difftime(time(0), updateStartTime) > 1) printf(" Slow place_id %d\n", place_id); if (difftime(time(0), updateStartTime) > 1) printf(" Slow place_id %d\n", place_id);

165
nominatim/partitionedtags.def
View File

@ -1,2 +1,165 @@
amenity pub amenity airport
amenity arts_centre
amenity atm
amenity auditorium
amenity bank
amenity bar
amenity bench
amenity bicycle_parking
amenity bicycle_rental
amenity brothel
amenity bureau_de_change
amenity bus_station
amenity cafe
amenity car_rental
amenity car_wash
amenity casino
amenity cinema
amenity clinic
amenity club
amenity college
amenity community_centre
amenity courthouse
amenity crematorium
amenity dentist
amenity doctors
amenity dormitory
amenity drinking_water
amenity driving_school
amenity embassy
amenity emergency_phone
amenity fast_food
amenity ferry_terminal
amenity fire_hydrant
amenity fire_station
amenity fountain
amenity fuel
amenity grave_yard
amenity hall
amenity health_centre
amenity hospital
amenity hotel amenity hotel
amenity hunting_stand
amenity ice_cream
amenity kindergarten
amenity library
amenity market
amenity marketplace
amenity nightclub
amenity nursery
amenity nursing_home
amenity office
amenity park
amenity parking
amenity pharmacy
amenity place_of_worship
amenity police
amenity post_box
amenity post_office
amenity preschool
amenity prison
amenity pub
amenity public_building
amenity public_market
amenity reception_area
amenity restaurant
amenity retirement_home
amenity sauna
amenity school
amenity shelter
amenity shop
amenity shopping
amenity social_club
amenity studio
amenity supermarket
amenity taxi
amenity telephone
amenity theatre
amenity toilets
amenity townhall
amenity university
amenity veterinary
amenity waste_basket
amenity wifi
amenity youth_centre
boundary administrative
building apartments
building block
building bunker
building chapel
building church
building commercial
building dormitory
building entrance
building faculty
building farm
building flats
building garage
building hospital
building hotel
building house
building industrial
building office
building public
building residential
building retail
building school
building shop
building stadium
building store
building terrace
building tower
building train_station
building university
highway bridleway
highway bus_stop
highway construction
highway cycleway
highway distance_marker
highway emergency_access_point
highway footway
highway gate
highway motorway_junction
highway path
highway pedestrian
highway platform
highway primary
highway primary_link
highway raceway
highway road
highway secondary
highway secondary_link
highway services
highway steps
highway tertiary
highway track
highway trail
highway trunk
highway trunk_link
highway unsurfaced
historic archaeological_site
historic battlefield
historic building
historic castle
historic church
historic house
historic icon
historic manor
historic memorial
historic mine
historic monument
historic museum
historic ruins
historic tower
historic wayside_cross
historic wayside_shrine
historic wreck
landuse cemetery
landuse commercial
landuse construction
landuse farm
landuse farmland
landuse farmyard
landuse forest
landuse grass
landuse industrial

1
settings/settings.php
View File

@ -1,5 +1,4 @@
<?php <?php
if (file_exists(CONST_BasePath.'/settings/local.php')) require_once(CONST_BasePath.'/settings/local.php'); if (file_exists(CONST_BasePath.'/settings/local.php')) require_once(CONST_BasePath.'/settings/local.php');
if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true); if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true);

60
sql/functions.sql
View File

@ -1966,18 +1966,23 @@ BEGIN
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename; INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
IF for_place_id IS NULL THEN IF for_place_id IS NULL THEN
select parent_place_id,'us', housenumber, 30, postcode, null from location_property_aux
WHERE place_id = in_place_id
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
END IF;
IF for_place_id IS NULL THEN
select parent_place_id, country_code, housenumber, rank_address, postcode, name from placex select parent_place_id, country_code, housenumber, rank_address, postcode, name from placex
WHERE place_id = in_place_id and rank_address = 30 WHERE place_id = in_place_id and rank_address = 30
INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename; INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
IF for_place_id IS NULL THEN
for_place_id := in_place_id;
select country_code, housenumber, rank_address, postcode, null from placex where place_id = for_place_id
INTO searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
END IF;
END IF; END IF;
IF for_place_id IS NULL THEN
for_place_id := in_place_id;
select country_code, housenumber, rank_address, postcode, null from placex where place_id = for_place_id
INTO searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
END IF;
--RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode; --RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode;
found := 1000; found := 1000;
@ -2355,3 +2360,44 @@ BEGIN
END; END;
$$ $$
LANGUAGE plpgsql; LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION aux_create_property(pointgeo GEOMETRY, in_housenumber TEXT,
in_street TEXT, in_isin TEXT, in_postcode TEXT, in_countrycode char(2)) RETURNS INTEGER
AS $$
DECLARE
newpoints INTEGER;
place_centroid GEOMETRY;
partition INTEGER;
parent_place_id INTEGER;
location RECORD;
address_street_word_id INTEGER;
BEGIN
place_centroid := ST_Centroid(pointgeo);
partition := get_partition(place_centroid, in_countrycode);
parent_place_id := null;
address_street_word_id := get_name_id(make_standard_name(in_street));
IF address_street_word_id IS NOT NULL THEN
FOR location IN SELECT * from getNearestNamedRoadFeature(partition, place_centroid, address_street_word_id) LOOP
parent_place_id := location.place_id;
END LOOP;
END IF;
IF parent_place_id IS NULL THEN
FOR location IN SELECT place_id FROM getNearestRoadFeature(partition, place_centroid) LOOP
parent_place_id := location.place_id;
END LOOP;
END IF;
newpoints := 0;
insert into location_property_aux (place_id, partition, parent_place_id, housenumber, postcode, centroid)
values (nextval('seq_place'), partition, parent_place_id, in_housenumber, in_postcode, place_centroid);
newpoints := newpoints + 1;
RETURN newpoints;
END;
$$
LANGUAGE plpgsql;

6
sql/tables.sql
View File

@ -100,6 +100,12 @@ CREATE TABLE location_property (
postcode TEXT postcode TEXT
); );
SELECT AddGeometryColumn('location_property', 'centroid', 4326, 'POINT', 2); SELECT AddGeometryColumn('location_property', 'centroid', 4326, 'POINT', 2);
CREATE TABLE location_property_aux () INHERITS (location_property);
CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
CREATE TABLE location_property_tiger () INHERITS (location_property); CREATE TABLE location_property_tiger () INHERITS (location_property);
CREATE INDEX idx_location_property_tiger_place_id ON location_property_tiger USING BTREE (place_id); CREATE INDEX idx_location_property_tiger_place_id ON location_property_tiger USING BTREE (place_id);
CREATE INDEX idx_location_property_tiger_parent_place_id ON location_property_tiger USING BTREE (parent_place_id); CREATE INDEX idx_location_property_tiger_parent_place_id ON location_property_tiger USING BTREE (parent_place_id);

11
utils/setup.php
View File

@ -318,7 +318,9 @@ var_dump($sPartitionName);
else passthru(CONST_BasePath.'/osmosis-0.38/bin/osmosis --read-replication-interval-init '.CONST_BasePath.'/settings'); else passthru(CONST_BasePath.'/osmosis-0.38/bin/osmosis --read-replication-interval-init '.CONST_BasePath.'/settings');
$sDate = $aCMDResult['osmosis-init-date']; $sDate = $aCMDResult['osmosis-init-date'];
$sStateFile = file_get_contents('http://toolserver.org/~mazder/replicate-sequences/?'.$sDate); $sURL = 'http://toolserver.org/~mazder/replicate-sequences/?'.$sDate;
echo "Getting state file: $sURL\n";
$sStateFile = file_get_contents($sURL);
if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file"); if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile); file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
} }
@ -380,22 +382,15 @@ var_dump($sPartitionName);
$hProcess = proc_open($sCMD, $aDescriptors, $ahPipes); $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
if (!is_resource($hProcess)) fail('unable to start pgsql'); if (!is_resource($hProcess)) fail('unable to start pgsql');
echo "write";
fwrite($ahPipes[0], $sScript); fwrite($ahPipes[0], $sScript);
echo "close";
fclose($ahPipes[0]); fclose($ahPipes[0]);
echo "done";
// TODO: error checking // TODO: error checking
while(!feof($ahPipes[1])) while(!feof($ahPipes[1]))
{ {
echo "read";
echo fread($ahPipes[1], 4096); echo fread($ahPipes[1], 4096);
} }
echo "done";
fclose($ahPipes[1]); fclose($ahPipes[1]);
echo "done1";
proc_close($hProcess); proc_close($hProcess);
echo "done2";
} }

8
utils/specialphrases.php
View File

@ -83,10 +83,11 @@
foreach($aPairs as $aPair) foreach($aPairs as $aPair)
{ {
if ($aPair[1] == 'highway') continue;
echo "create table place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." as "; echo "create table place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." as ";
echo "select place_id as place_id,st_centroid(geometry) as centroid from placex where "; echo "select place_id as place_id,st_centroid(geometry) as centroid from placex where ";
echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."';\n"; echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."' limit 0;\n";
echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_centroid "; echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_centroid ";
echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING GIST (centroid);\n"; echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING GIST (centroid);\n";
@ -94,5 +95,10 @@
echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_place_id "; echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_place_id ";
echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING btree(place_id);\n"; echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING btree(place_id);\n";
echo "truncate place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1]).";\n";
echo "insert into place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." ";
echo "select place_id as place_id,st_centroid(geometry) as centroid from placex where ";
echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."';\n";
} }
} }

2
website/details.php
View File

@ -30,6 +30,8 @@
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID); $iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID; if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
$aLangPrefOrder = getPrefferedLangauges(); $aLangPrefOrder = getPrefferedLangauges();
$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]"; $sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]";

122
website/search.php
View File

@ -35,7 +35,7 @@
// Prefered language // Prefered language
$aLangPrefOrder = getPrefferedLangauges(); $aLangPrefOrder = getPrefferedLangauges();
if (isset($aLangPrefOrder['name:de'])) $bReverseInPlan = true; // if (isset($aLangPrefOrder['name:de'])) $bReverseInPlan = true;
$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]"; $sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]";
if (isset($_GET['exclude_place_ids']) && $_GET['exclude_place_ids']) if (isset($_GET['exclude_place_ids']) && $_GET['exclude_place_ids'])
@ -63,6 +63,9 @@
{ {
$hLog = logStart($oDB, 'search', $sQuery, $aLangPrefOrder); $hLog = logStart($oDB, 'search', $sQuery, $aLangPrefOrder);
// Hack to make it handle "new york, ny" (and variants) correctly
$sQuery = str_ireplace(array('New York, ny','new york, new york', 'New York ny','new york new york'), 'new york city, new york', $sQuery);
// If we have a view box create the SQL // If we have a view box create the SQL
// Small is the actual view box, Large is double (on each axis) that // Small is the actual view box, Large is double (on each axis) that
$sViewboxCentreSQL = $sViewboxSmallSQL = $sViewboxLargeSQL = false; $sViewboxCentreSQL = $sViewboxSmallSQL = $sViewboxLargeSQL = false;
@ -151,7 +154,7 @@
// Start with a blank search // Start with a blank search
$aSearches = array( $aSearches = array(
array('iSearchRank' => 0, 'iNamePhrase' => 0, 'sCountryCode' => false, 'aName'=>array(), 'aAddress'=>array(), array('iSearchRank' => 0, 'iNamePhrase' => -1, 'sCountryCode' => false, 'aName'=>array(), 'aAddress'=>array(),
'sOperator'=>'', 'aFeatureName' => array(), 'sClass'=>'', 'sType'=>'', 'sHouseNumber'=>'', 'fLat'=>'', 'fLon'=>'', 'fRadius'=>'') 'sOperator'=>'', 'aFeatureName' => array(), 'sClass'=>'', 'sType'=>'', 'sHouseNumber'=>'', 'fLat'=>'', 'fLon'=>'', 'fRadius'=>'')
); );
@ -180,7 +183,7 @@
$sQuery = str_replace($aSpecialTerm[0], ' ', $sQuery); $sQuery = str_replace($aSpecialTerm[0], ' ', $sQuery);
$sToken = $oDB->getOne("select make_standard_name('".$aSpecialTerm[1]."') as string"); $sToken = $oDB->getOne("select make_standard_name('".$aSpecialTerm[1]."') as string");
$sSQL = 'select * from (select word_id,word_token, word, class, type, location, country_code, operator'; $sSQL = 'select * from (select word_id,word_token, word, class, type, location, country_code, operator';
$sSQL .= ' from word where word_token in (\' '.$sToken.'\')) as x where (class is not null and class != \'place\') or country_code is not null'; $sSQL .= ' from word where word_token in (\' '.$sToken.'\')) as x where (class is not null and class not in (\'place\',\'highway\')) or country_code is not null';
$aSearchWords = $oDB->getAll($sSQL); $aSearchWords = $oDB->getAll($sSQL);
$aNewSearches = array(); $aNewSearches = array();
foreach($aSearches as $aSearch) foreach($aSearches as $aSearch)
@ -245,7 +248,8 @@
// Check which tokens we have, get the ID numbers // Check which tokens we have, get the ID numbers
$sSQL = 'select word_id,word_token, word, class, type, location, country_code, operator'; $sSQL = 'select word_id,word_token, word, class, type, location, country_code, operator';
$sSQL .= ' from word where word_token in ('.join(',',array_map("getDBQuoted",$aTokens)).')'; $sSQL .= ' from word where word_token in ('.join(',',array_map("getDBQuoted",$aTokens)).')';
// $sSQL .= ' group by word_token, word, class, type, location,country_code'; $sSQL .= ' and (class is null or class not in (\'highway\'))';
// $sSQL .= ' group by word_token, word, class, type, location, country_code';
if (CONST_Debug) var_Dump($sSQL); if (CONST_Debug) var_Dump($sSQL);
@ -349,10 +353,15 @@
// Add all words from this wordset // Add all words from this wordset
foreach($aWordset as $sToken) foreach($aWordset as $sToken)
{ {
//echo "<br><b>$sToken</b>";
$aNewWordsetSearches = array(); $aNewWordsetSearches = array();
foreach($aWordsetSearches as $aCurrentSearch) foreach($aWordsetSearches as $aCurrentSearch)
{ {
//echo "<i>";
//var_dump($aCurrentSearch);
//echo "</i>";
// If the token is valid // If the token is valid
if (isset($aValidTokens[' '.$sToken])) if (isset($aValidTokens[' '.$sToken]))
{ {
@ -423,31 +432,47 @@
{ {
if (sizeof($aSearch['aName'])) if (sizeof($aSearch['aName']))
{ {
$aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id']; if (!isset($aValidTokens[$sToken]) || strlen($sToken) < 4)
{
$aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
}
else
{
$aSearch['iSearchRank'] += 1000; // skip;
}
} }
else else
{ {
$aSearch['aName'][$aSearchTerm['word_id']] = $aSearchTerm['word_id']; $aSearch['aName'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
$aSearch['iNamePhrase'] = $iPhrase; // $aSearch['iNamePhrase'] = $iPhrase;
} }
if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch; if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
} }
} }
} }
if (isset($aValidTokens[$sToken])) if (isset($aValidTokens[$sToken]) && strlen($sToken) >= 4)
{ {
// Allow searching for a word - but at extra cost // Allow searching for a word - but at extra cost
foreach($aValidTokens[$sToken] as $aSearchTerm) foreach($aValidTokens[$sToken] as $aSearchTerm)
{ {
$aSearch = $aCurrentSearch; //var_Dump('<hr>',$aSearch['aName']);
$aSearch['iSearchRank']+=5;
// $aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id']; if (sizeof($aCurrentSearch['aName']))
if (!sizeof($aSearch['aName']) || $aSearch['iNamePhrase'] == $iPhrase)
{ {
$aSearch = $aCurrentSearch;
$aSearch['iSearchRank'] += 1;
$aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
}
if (!sizeof($aCurrentSearch['aName']) || $aCurrentSearch['iNamePhrase'] == $iPhrase)
{
$aSearch = $aCurrentSearch;
$aSearch['iSearchRank'] += 4;
$aSearch['aName'][$aSearchTerm['word_id']] = $aSearchTerm['word_id']; $aSearch['aName'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
$aSearch['iNamePhrase'] = $iPhrase; $aSearch['iNamePhrase'] = $iPhrase;
if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
} }
if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
} }
} }
else else
@ -489,6 +514,9 @@
$aSearches = array_merge($aSearches, $aNewSearches); $aSearches = array_merge($aSearches, $aNewSearches);
if ($iSearchCount > 50) break; if ($iSearchCount > 50) break;
} }
// if (CONST_Debug) _debugDumpGroupedSearches($aGroupedSearches, $aValidTokens);
} }
} }
else else
@ -508,47 +536,7 @@
if (CONST_Debug) var_Dump($aGroupedSearches); if (CONST_Debug) var_Dump($aGroupedSearches);
if ($bReverseInPlan) if ($bReverseInPlan && false)
{
foreach($aGroupedSearches as $iGroup => $aSearches)
{
foreach($aSearches as $iSearch => $aSearch)
{
if (sizeof($aSearch['aAddress']))
{
$aReverseSearch = $aSearch;
$iReverseItem = array_pop($aSearch['aAddress']);
$aReverseSearch['aName'][$iReverseItem] = $iReverseItem;
$aGroupedSearches[$iGroup][] = $aReverseSearch;
}
}
}
}
//var_Dump($aGroupedSearches); exit;
// Filter out duplicate searches
$aSearchHash = array();
foreach($aGroupedSearches as $iGroup => $aSearches)
{
foreach($aSearches as $iSearch => $aSearch)
{
$sHash = serialize($aSearch);
if (isset($aSearchHash[$sHash]))
{
unset($aGroupedSearches[$iGroup][$iSearch]);
if (sizeof($aGroupedSearches[$iGroup]) == 0) unset($aGroupedSearches[$iGroup]);
}
else
{
$aSearchHash[$sHash] = 1;
}
}
}
if (CONST_Debug) _debugDumpGroupedSearches($aGroupedSearches, $aValidTokens);
if ($bReverseInPlan)
{ {
foreach($aGroupedSearches as $iGroup => $aSearches) foreach($aGroupedSearches as $iGroup => $aSearches)
{ {
@ -715,7 +703,19 @@
if (CONST_Debug) var_dump($sSQL); if (CONST_Debug) var_dump($sSQL);
$aPlaceIDs = $oDB->getCol($sSQL); $aPlaceIDs = $oDB->getCol($sSQL);
// If not try the tiger fallback table // If not try the aux fallback table
if (!sizeof($aPlaceIDs))
{
$sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
if (sizeof($aExcludePlaceIDs))
{
$sSQL .= " and place_id not in (".join(',',$aExcludePlaceIDs).")";
}
// $sSQL .= " limit $iLimit";
if (CONST_Debug) var_dump($sSQL);
$aPlaceIDs = $oDB->getCol($sSQL);
}
if (!sizeof($aPlaceIDs)) if (!sizeof($aPlaceIDs))
{ {
$sSQL = "select place_id from location_property_tiger where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'"; $sSQL = "select place_id from location_property_tiger where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
@ -846,7 +846,7 @@
$sSQL .= ",get_name_by_language(name, ARRAY['ref']) "; $sSQL .= ",get_name_by_language(name, ARRAY['ref']) ";
$sSQL .= " union "; $sSQL .= " union ";
$sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,"; $sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,";
$sSQL .= "get_tiger_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
$sSQL .= "null as placename,"; $sSQL .= "null as placename,";
$sSQL .= "null as ref,"; $sSQL .= "null as ref,";
$sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
@ -854,7 +854,17 @@
$sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) "; $sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) ";
$sSQL .= "group by place_id"; $sSQL .= "group by place_id";
if (!$bDeDupe) $sSQL .= ",place_id"; if (!$bDeDupe) $sSQL .= ",place_id";
$sSQL .= ",get_tiger_address_by_language(place_id, $sLanguagePrefArraySQL) "; $sSQL .= " union ";
$sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,";
$sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
$sSQL .= "null as placename,";
$sSQL .= "null as ref,";
$sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
$sSQL .= $sOrderSQL." as porder ";
$sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
$sSQL .= "group by place_id";
if (!$bDeDupe) $sSQL .= ",place_id";
$sSQL .= ",get_address_by_language(place_id, $sLanguagePrefArraySQL) ";
$sSQL .= "order by rank_search,rank_address,porder asc"; $sSQL .= "order by rank_search,rank_address,porder asc";
if (CONST_Debug) var_dump('<hr>',$sSQL); if (CONST_Debug) var_dump('<hr>',$sSQL);
$aSearchResults = $oDB->getAll($sSQL); $aSearchResults = $oDB->getAll($sSQL);