aux properties, memory leaks, tweaks to search order

nominatim/export.c

@@ -161,7 +161,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
 
     pg_prepare_params[0] = PG_OID_INT8;
     res = PQprepare(conn, "placex_address",
-                    "select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc",
+                    "select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc,osm_type,osm_id",
                     1, pg_prepare_params);
     if (PQresultStatus(res) != PGRES_COMMAND_OK)
     {
@@ -172,7 +172,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
 
     pg_prepare_params[0] = PG_OID_INT8;
     res = PQprepare(conn, "placex_names",
-                    "select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x",
+                    "select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x order by (each(name)).key",
                     1, pg_prepare_params);
     if (PQresultStatus(res) != PGRES_COMMAND_OK)
     {
@@ -183,7 +183,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
 
     pg_prepare_params[0] = PG_OID_INT8;
     res = PQprepare(conn, "placex_extratags",
-                    "select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x",
+                    "select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x order by (each(extratags)).key",
                     1, pg_prepare_params);
     if (PQresultStatus(res) != PGRES_COMMAND_OK)
     {
@@ -402,6 +402,7 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
     PQclear(res);
     PQclear(resNames);
     PQclear(resAddress);
+    PQclear(resExtraTags);
 }
 
 const char * getRankLabel(int rank)

nominatim/import.c

@@ -16,11 +16,11 @@
 typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
 typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
 
-#define MAX_FEATUREADDRESS 500
+#define MAX_FEATUREADDRESS 5000
-#define MAX_FEATURENAMES 1000
+#define MAX_FEATURENAMES 10000
-#define MAX_FEATUREEXTRATAGS 100
+#define MAX_FEATUREEXTRATAGS 10000
-#define MAX_FEATURENAMESTRING 100000
+#define MAX_FEATURENAMESTRING 1000000
-#define MAX_FEATUREEXTRATAGSTRING 50000
+#define MAX_FEATUREEXTRATAGSTRING 500000
 
 struct feature_address
 {
@@ -42,7 +42,7 @@ struct feature_tag
 
 struct feature
 {
-    int			placeID;
+    xmlChar *   placeID;
     xmlChar *	type;
     xmlChar *	id;
     xmlChar *	key;
@@ -92,7 +92,7 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
             }
             else
             {
-                fprintf( stderr, "Unknown osmStructured version %f\n", version );
+                fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
                 exit_nicely();
             }
         }
@@ -127,6 +127,7 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
 
     if (xmlStrEqual(name, BAD_CAST "feature"))
     {
+        feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
         feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
         feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
         feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
@@ -140,32 +141,39 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
         feature.geometry = NULL;
         featureAddressLines = 0;
         featureNameLines = 0;
+		featureExtraTagLines = 0;
 
         return;
     }
     if (xmlStrEqual(name, BAD_CAST "names")) return;
     if (xmlStrEqual(name, BAD_CAST "name"))
     {
-        featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
+        if (featureNameLines < MAX_FEATURENAMES)
-        featureName[featureNameLines].value = xmlTextReaderReadString(reader);
-        featureNameLines++;
-        if (featureNameLines >= MAX_FEATURENAMES)
         {
-            fprintf( stderr, "Too many name elements\n");
+	        featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
-            exit_nicely();
+    	    featureName[featureNameLines].value = xmlTextReaderReadString(reader);
+        	featureNameLines++;
+		}
+		else
+		{
+            fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
+//            exit_nicely();
         }
         return;
     }
     if (xmlStrEqual(name, BAD_CAST "tags")) return;
     if (xmlStrEqual(name, BAD_CAST "tag"))
     {
-        featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
+        if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
-        featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
+		{
-        featureExtraTagLines++;
+	        featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
-        if (featureExtraTagLines >= MAX_FEATUREEXTRATAGS)
+    	    featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
+        	featureExtraTagLines++;
+		}
+		else
         {
-            fprintf( stderr, "Too many extra tag elements\n");
+            fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
-            exit_nicely();
+//            exit_nicely();
         }
         return;
     }
@@ -261,36 +269,39 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
     }
     if (isAddressLine)
     {
-        value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
+        if (featureAddressLines < MAX_FEATUREADDRESS)
-        if (!value)
+		{
-        {
+	        value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
-            fprintf( stderr, "Address element missing rank\n");
+    	    if (!value)
-            exit_nicely();
+	        {
-        }
+    	        fprintf( stderr, "Address element missing rank\n");
-        featureAddress[featureAddressLines].rankAddress =  atoi(value);
+        	    exit_nicely();
-        xmlFree(value);
+	        }
+    	    featureAddress[featureAddressLines].rankAddress =  atoi(value);
+        	xmlFree(value);
 
-        value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
+	        value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
-        if (!value)
+    	    if (!value)
-        {
+        	{
-            fprintf( stderr, "Address element missing rank\n");
+	            fprintf( stderr, "Address element missing rank\n");
-            exit_nicely();
+    	        exit_nicely();
-        }
+        	}
-        if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
+	        if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
-        else strcpy(featureAddress[featureAddressLines].isAddress, "f");
+    	    else strcpy(featureAddress[featureAddressLines].isAddress, "f");
-        xmlFree(value);
+        	xmlFree(value);
 
-        featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
+	        featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
-        featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
+    	    featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
-        featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
+        	featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
-        featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
+	        featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
-        featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
+    	    featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
 	
-        featureAddressLines++;
+    	    featureAddressLines++;
-        if (featureAddressLines >= MAX_FEATUREADDRESS)
+		}
+		else
         {
-            fprintf( stderr, "Too many address elements\n");
+            fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
-            exit_nicely();
+//            exit_nicely();
         }
 
         return;
@@ -311,7 +322,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
     {
         featureCount++;
         if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
-
+/*
         if (fileMode == FILEMODE_ADD)
         {
             resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
@@ -336,7 +347,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
                 exit(EXIT_FAILURE);
             }
         }
-        place_id = PQgetvalue(resPlaceID, 0, 0);
+*/
+        place_id = feature.placeID;
 
         if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE)
         {
@@ -517,6 +529,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
             }
         }
 
+        xmlFree(feature.placeID);
         xmlFree(feature.type);
         xmlFree(feature.id);
         xmlFree(feature.key);
@@ -529,7 +542,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
         if (feature.houseNumber) xmlFree(feature.houseNumber);
         if (feature.geometry) xmlFree(feature.geometry);
 
-        PQclear(resPlaceID);
+//        PQclear(resPlaceID);
     }
 }
 

nominatim/index.c

@@ -157,9 +157,9 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
         paramValues[0] = (char *)&paramRank;
         paramLengths[0] = sizeof(paramRank);
         paramFormats[0] = 1;
-        if (rank < 16)
+//        if (rank < 16)
-            resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
+//            resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
-        else
+//        else
             resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
         if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
         {
@@ -226,10 +226,14 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
                 paramValues[1] = (char *)&paramSector;
                 paramLengths[1] = sizeof(paramSector);
                 paramFormats[1] = 1;
-                if (rank < 16)
+                if (rankTotalTuples-rankCountTuples < num_threads*20)
-                    iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
+		{
+			iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
+		}
                 else
+		{
                     iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
+		}
                 if (!iResult)
                 {
                     fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
@@ -288,6 +292,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
 
                 PQclear(resPlaces);
             }
+            if (rankTotalTuples-rankCountTuples < num_threads*20) iSector = PQntuples(resSectors);
         }
         // Finished rank
         printf("\r  Done %i in %i @ %f per second - FINISHED                      \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
@@ -331,16 +336,30 @@ void *nominatim_indexThread(void * thread_data_in)
         if (verbose) printf("  Processing place_id %d\n", place_id);
 
         updateStartTime = time(0);
-        paramPlaceID = PGint32(place_id);
+	int done = 0;
-        paramValues[0] = (char *)&paramPlaceID;
+	while(!done)
-        paramLengths[0] = sizeof(paramPlaceID);
+	{
-        paramFormats[0] = 1;
+	        paramPlaceID = PGint32(place_id);
-        res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1);
+        	paramValues[0] = (char *)&paramPlaceID;
-        if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	        paramLengths[0] = sizeof(paramPlaceID);
-        {
+        	paramFormats[0] = 1;
-            fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
+	        res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1);
-            PQclear(res);
+        	if (PQresultStatus(res) == PGRES_COMMAND_OK)
-            exit(EXIT_FAILURE);
+			done = 1;
+		else
+	        {
+			if (strncmp(PQerrorMessage(thread_data->conn), "ERROR:  deadlock detected", 25))
+			{
+		            fprintf(stderr, "index_placex: UPDATE failed - deadlock, retrying\n");
+			}
+			else
+			{
+		            fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
+		            PQclear(res);
+				sleep(5);
+//		            exit(EXIT_FAILURE);
+			}
+		}
         }
         PQclear(res);
         if (difftime(time(0), updateStartTime) > 1) printf("  Slow place_id %d\n", place_id);

nominatim/partitionedtags.def

@@ -1,2 +1,165 @@
-amenity	pub
+amenity airport
+amenity arts_centre
+amenity atm
+amenity auditorium
+amenity bank
+amenity bar
+amenity bench
+amenity bicycle_parking
+amenity bicycle_rental
+amenity brothel
+amenity bureau_de_change
+amenity bus_station
+amenity cafe
+amenity car_rental
+amenity car_wash
+amenity casino
+amenity cinema
+amenity clinic
+amenity club
+amenity college
+amenity community_centre
+amenity courthouse
+amenity crematorium
+amenity dentist
+amenity doctors
+amenity dormitory
+amenity drinking_water
+amenity driving_school
+amenity embassy
+amenity emergency_phone
+amenity fast_food
+amenity ferry_terminal
+amenity fire_hydrant
+amenity fire_station
+amenity fountain
+amenity fuel
+amenity grave_yard
+amenity hall
+amenity health_centre
+amenity hospital
 amenity hotel
+amenity hunting_stand
+amenity ice_cream
+amenity kindergarten
+amenity library
+amenity market
+amenity marketplace
+amenity nightclub
+amenity nursery
+amenity nursing_home
+amenity office
+amenity park
+amenity parking
+amenity pharmacy
+amenity place_of_worship
+amenity police
+amenity post_box
+amenity post_office
+amenity preschool
+amenity prison
+amenity pub
+amenity public_building
+amenity public_market
+amenity reception_area
+amenity restaurant
+amenity retirement_home
+amenity sauna
+amenity school
+amenity shelter
+amenity shop
+amenity shopping
+amenity social_club
+amenity studio
+amenity supermarket
+amenity taxi
+amenity telephone
+amenity theatre
+amenity toilets
+amenity townhall
+amenity university
+amenity veterinary
+amenity waste_basket
+amenity wifi
+amenity youth_centre
+boundary administrative
+building apartments
+building block
+building bunker
+building chapel
+building church
+building commercial
+building dormitory
+building entrance
+building faculty
+building farm
+building flats
+building garage
+building hospital
+building hotel
+building house
+building industrial
+building office
+building public
+building residential
+building retail
+building school
+building shop
+building stadium
+building store
+building terrace
+building tower
+building train_station
+building university
+highway bridleway
+highway bus_stop
+highway construction
+highway cycleway
+highway distance_marker
+highway emergency_access_point
+highway footway
+highway gate
+highway motorway_junction
+highway path
+highway pedestrian
+highway platform
+highway primary
+highway primary_link
+highway raceway
+highway road
+highway secondary
+highway secondary_link
+highway services
+highway steps
+highway tertiary
+highway track
+highway trail
+highway trunk
+highway trunk_link
+highway unsurfaced
+historic archaeological_site
+historic battlefield
+historic building
+historic castle
+historic church
+historic house
+historic icon
+historic manor
+historic memorial
+historic mine
+historic monument
+historic museum
+historic ruins
+historic tower
+historic wayside_cross
+historic wayside_shrine
+historic wreck
+landuse cemetery
+landuse commercial
+landuse construction
+landuse farm
+landuse farmland
+landuse farmyard
+landuse forest
+landuse grass
+landuse industrial

settings/settings.php

@@ -1,5 +1,4 @@
 <?php
-
 	if (file_exists(CONST_BasePath.'/settings/local.php')) require_once(CONST_BasePath.'/settings/local.php');
 	if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true);
 

sql/functions.sql

@@ -1966,18 +1966,23 @@ BEGIN
     INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
 
   IF for_place_id IS NULL THEN
+    select parent_place_id,'us', housenumber, 30, postcode, null from location_property_aux
+      WHERE place_id = in_place_id 
+      INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
+  END IF;
 
+  IF for_place_id IS NULL THEN
     select parent_place_id, country_code, housenumber, rank_address, postcode, name from placex 
       WHERE place_id = in_place_id and rank_address = 30 
       INTO for_place_id, searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
-
-    IF for_place_id IS NULL THEN
-      for_place_id := in_place_id;
-      select country_code, housenumber, rank_address, postcode, null from placex where place_id = for_place_id 
-        INTO searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
-    END IF;
-
   END IF;
+
+  IF for_place_id IS NULL THEN
+    for_place_id := in_place_id;
+    select country_code, housenumber, rank_address, postcode, null from placex where place_id = for_place_id 
+      INTO searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename;
+  END IF;
+
 --RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode;
 
   found := 1000;
@@ -2355,3 +2360,44 @@ BEGIN
 END;
 $$
 LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION aux_create_property(pointgeo GEOMETRY, in_housenumber TEXT, 
+  in_street TEXT, in_isin TEXT, in_postcode TEXT, in_countrycode char(2)) RETURNS INTEGER
+  AS $$
+DECLARE
+
+  newpoints INTEGER;
+  place_centroid GEOMETRY;
+  partition INTEGER;
+  parent_place_id INTEGER;
+  location RECORD;
+  address_street_word_id INTEGER;  
+
+BEGIN
+
+  place_centroid := ST_Centroid(pointgeo);
+  partition := get_partition(place_centroid, in_countrycode);
+  parent_place_id := null;
+
+  address_street_word_id := get_name_id(make_standard_name(in_street));
+  IF address_street_word_id IS NOT NULL THEN
+    FOR location IN SELECT * from getNearestNamedRoadFeature(partition, place_centroid, address_street_word_id) LOOP
+      parent_place_id := location.place_id;
+    END LOOP;
+  END IF;
+
+  IF parent_place_id IS NULL THEN
+    FOR location IN SELECT place_id FROM getNearestRoadFeature(partition, place_centroid) LOOP
+      parent_place_id := location.place_id;
+    END LOOP;    
+  END IF;
+
+  newpoints := 0;
+  insert into location_property_aux (place_id, partition, parent_place_id, housenumber, postcode, centroid)
+    values (nextval('seq_place'), partition, parent_place_id, in_housenumber, in_postcode, place_centroid);
+  newpoints := newpoints + 1;
+
+  RETURN newpoints;
+END;
+$$
+LANGUAGE plpgsql;

sql/tables.sql

@@ -100,6 +100,12 @@ CREATE TABLE location_property (
   postcode TEXT
   );
 SELECT AddGeometryColumn('location_property', 'centroid', 4326, 'POINT', 2);
+
+CREATE TABLE location_property_aux () INHERITS (location_property);
+CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
+CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
+CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
+
 CREATE TABLE location_property_tiger () INHERITS (location_property);
 CREATE INDEX idx_location_property_tiger_place_id ON location_property_tiger USING BTREE (place_id);
 CREATE INDEX idx_location_property_tiger_parent_place_id ON location_property_tiger USING BTREE (parent_place_id);

utils/setup.php

@@ -318,7 +318,9 @@ var_dump($sPartitionName);
 		else passthru(CONST_BasePath.'/osmosis-0.38/bin/osmosis --read-replication-interval-init '.CONST_BasePath.'/settings');
 
 		$sDate = $aCMDResult['osmosis-init-date'];
-		$sStateFile = file_get_contents('http://toolserver.org/~mazder/replicate-sequences/?'.$sDate);
+		$sURL = 'http://toolserver.org/~mazder/replicate-sequences/?'.$sDate;
+		echo "Getting state file: $sURL\n";
+		$sStateFile = file_get_contents($sURL);
 		if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
 		file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
 	}
@@ -380,22 +382,15 @@ var_dump($sPartitionName);
 		$hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 		if (!is_resource($hProcess)) fail('unable to start pgsql');
 
-echo "write";
 		fwrite($ahPipes[0], $sScript);
-echo "close";
 		fclose($ahPipes[0]);
-echo "done";
 
 		// TODO: error checking
 		while(!feof($ahPipes[1]))
 		{
-echo "read";
 			echo fread($ahPipes[1], 4096);
 		}
-echo "done";
 		fclose($ahPipes[1]);
 
-echo "done1";
 		proc_close($hProcess);
-echo "done2";
 	}

utils/specialphrases.php

@@ -83,10 +83,11 @@
 
 		foreach($aPairs as $aPair)
 		{
+			if ($aPair[1] == 'highway') continue;
 
 			echo "create table place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." as ";
 			echo "select place_id as place_id,st_centroid(geometry) as centroid from placex where ";
-			echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."';\n";
+			echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."' limit 0;\n";
 
 			echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_centroid ";
 			echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING GIST (centroid);\n";
@@ -94,5 +95,10 @@
 			echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_place_id ";
 			echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING btree(place_id);\n";
 
+			echo "truncate place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1]).";\n";
+			echo "insert into place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." ";
+			echo "select place_id as place_id,st_centroid(geometry) as centroid from placex where ";
+			echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."';\n";
+
 		}
 	}

website/details.php

@@ -30,6 +30,8 @@
 
 	$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
 	if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+	$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
+	if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
 
 	$aLangPrefOrder = getPrefferedLangauges();
 	$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]";

website/search.php

@@ -35,7 +35,7 @@
 
 	// Prefered language	
 	$aLangPrefOrder = getPrefferedLangauges();
-	if (isset($aLangPrefOrder['name:de'])) $bReverseInPlan = true;
+//	if (isset($aLangPrefOrder['name:de'])) $bReverseInPlan = true;
 	$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]";
 
 	if (isset($_GET['exclude_place_ids']) && $_GET['exclude_place_ids'])
@@ -63,6 +63,9 @@
 	{
 		$hLog = logStart($oDB, 'search', $sQuery, $aLangPrefOrder);
 
+		// Hack to make it handle "new york, ny" (and variants) correctly
+                $sQuery = str_ireplace(array('New York, ny','new york, new york', 'New York ny','new york new york'), 'new york city, new york', $sQuery);
+
 		// If we have a view box create the SQL
 		// Small is the actual view box, Large is double (on each axis) that 
 		$sViewboxCentreSQL = $sViewboxSmallSQL = $sViewboxLargeSQL = false;
@@ -151,7 +154,7 @@
 
 			// Start with a blank search
 			$aSearches = array(
-				array('iSearchRank' => 0, 'iNamePhrase' => 0, 'sCountryCode' => false, 'aName'=>array(), 'aAddress'=>array(), 
+				array('iSearchRank' => 0, 'iNamePhrase' => -1, 'sCountryCode' => false, 'aName'=>array(), 'aAddress'=>array(), 
 					'sOperator'=>'', 'aFeatureName' => array(), 'sClass'=>'', 'sType'=>'', 'sHouseNumber'=>'', 'fLat'=>'', 'fLon'=>'', 'fRadius'=>'')
 			);
 
@@ -180,7 +183,7 @@
 				$sQuery = str_replace($aSpecialTerm[0], ' ', $sQuery);
 				$sToken = $oDB->getOne("select make_standard_name('".$aSpecialTerm[1]."') as string");
 				$sSQL = 'select * from (select word_id,word_token, word, class, type, location, country_code, operator';
-				$sSQL .= ' from word where word_token in (\' '.$sToken.'\')) as x where (class is not null and class != \'place\') or country_code is not null';
+				$sSQL .= ' from word where word_token in (\' '.$sToken.'\')) as x where (class is not null and class not in (\'place\',\'highway\')) or country_code is not null';
 				$aSearchWords = $oDB->getAll($sSQL);
 				$aNewSearches = array();
 				foreach($aSearches as $aSearch)
@@ -245,7 +248,8 @@
 			// Check which tokens we have, get the ID numbers			
 			$sSQL = 'select word_id,word_token, word, class, type, location, country_code, operator';
 			$sSQL .= ' from word where word_token in ('.join(',',array_map("getDBQuoted",$aTokens)).')';
-//			$sSQL .= ' group by word_token, word, class, type, location,country_code';
+			$sSQL .= ' and (class is null or class not in (\'highway\'))';
+//			$sSQL .= ' group by word_token, word, class, type, location, country_code';
 
 			if (CONST_Debug) var_Dump($sSQL);
 
@@ -349,10 +353,15 @@
 						// Add all words from this wordset
 						foreach($aWordset as $sToken)
 						{
+//echo "<br><b>$sToken</b>";
 							$aNewWordsetSearches = array();
 							
 							foreach($aWordsetSearches as $aCurrentSearch)
 							{
+//echo "<i>";
+//var_dump($aCurrentSearch);
+//echo "</i>";
+
 								// If the token is valid
 								if (isset($aValidTokens[' '.$sToken]))
 								{
@@ -423,31 +432,47 @@
 										{
 											if (sizeof($aSearch['aName']))
 											{
-												$aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
+												if (!isset($aValidTokens[$sToken]) || strlen($sToken) < 4)
+												{
+													$aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
+												}
+												else
+												{
+													$aSearch['iSearchRank'] += 1000; // skip;
+												}
 											}
 											else
 											{
 												$aSearch['aName'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
-												$aSearch['iNamePhrase'] = $iPhrase;
+//												$aSearch['iNamePhrase'] = $iPhrase;
 											}
 											if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
 										}
 									}
 								}
-								if (isset($aValidTokens[$sToken]))
+								if (isset($aValidTokens[$sToken]) && strlen($sToken) >= 4)
 								{
 									// Allow searching for a word - but at extra cost
 									foreach($aValidTokens[$sToken] as $aSearchTerm)
 									{
-										$aSearch = $aCurrentSearch;
+//var_Dump('<hr>',$aSearch['aName']);
-										$aSearch['iSearchRank']+=5;
+
-//										$aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
+										if (sizeof($aCurrentSearch['aName']))
-										if (!sizeof($aSearch['aName']) || $aSearch['iNamePhrase'] == $iPhrase)
 										{
+  										$aSearch = $aCurrentSearch;
+											$aSearch['iSearchRank'] += 1;
+											$aSearch['aAddress'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
+  										if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
+										}
+
+										if (!sizeof($aCurrentSearch['aName']) || $aCurrentSearch['iNamePhrase'] == $iPhrase)
+										{
+  										$aSearch = $aCurrentSearch;
+											$aSearch['iSearchRank'] += 4;
 											$aSearch['aName'][$aSearchTerm['word_id']] = $aSearchTerm['word_id'];
 											$aSearch['iNamePhrase'] = $iPhrase;
+  										if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
 										}
-										if ($aSearch['iSearchRank'] < $iMaxRank) $aNewWordsetSearches[] = $aSearch;
 									}
 								}
 								else
@@ -489,6 +514,9 @@
 						$aSearches = array_merge($aSearches, $aNewSearches);
 						if ($iSearchCount > 50) break;
 					}
+
+//					if (CONST_Debug) _debugDumpGroupedSearches($aGroupedSearches, $aValidTokens);
+
 				}
 			}
 			else
@@ -508,47 +536,7 @@
 				
 				if (CONST_Debug) var_Dump($aGroupedSearches);
 
-				if ($bReverseInPlan)
+				if ($bReverseInPlan && false)
-				{
-					foreach($aGroupedSearches as $iGroup => $aSearches)
-					{
-						foreach($aSearches as $iSearch => $aSearch)
-						{
-							if (sizeof($aSearch['aAddress']))
-							{
-								$aReverseSearch = $aSearch;
-								$iReverseItem = array_pop($aSearch['aAddress']);
-								$aReverseSearch['aName'][$iReverseItem] = $iReverseItem;
-								$aGroupedSearches[$iGroup][] = $aReverseSearch;
-							}
-						}
-					}
-				}
-
-//var_Dump($aGroupedSearches); exit;
-
-				// Filter out duplicate searches
-				$aSearchHash = array();
-				foreach($aGroupedSearches as $iGroup => $aSearches)
-				{
-					foreach($aSearches as $iSearch => $aSearch)
-					{
-						$sHash = serialize($aSearch);
-						if (isset($aSearchHash[$sHash]))
-						{
-							unset($aGroupedSearches[$iGroup][$iSearch]);
-							if (sizeof($aGroupedSearches[$iGroup]) == 0) unset($aGroupedSearches[$iGroup]);
-						}
-						else
-						{
-							$aSearchHash[$sHash] = 1;
-						}
-					}
-				}
-				
-				if (CONST_Debug) _debugDumpGroupedSearches($aGroupedSearches, $aValidTokens);
-
-				if ($bReverseInPlan)
 				{
 					foreach($aGroupedSearches as $iGroup => $aSearches)
 					{
@@ -715,7 +703,19 @@
 								if (CONST_Debug) var_dump($sSQL);
 								$aPlaceIDs = $oDB->getCol($sSQL);
 
-								// If not try the tiger fallback table
+								// If not try the aux fallback table
+								if (!sizeof($aPlaceIDs))
+								{
+									$sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
+									if (sizeof($aExcludePlaceIDs))
+									{
+										$sSQL .= " and place_id not in (".join(',',$aExcludePlaceIDs).")";
+									}
+//									$sSQL .= " limit $iLimit";
+									if (CONST_Debug) var_dump($sSQL);
+									$aPlaceIDs = $oDB->getCol($sSQL);
+								}
+
 								if (!sizeof($aPlaceIDs))
 								{
 									$sSQL = "select place_id from location_property_tiger where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
@@ -846,7 +846,7 @@
 					$sSQL .= ",get_name_by_language(name, ARRAY['ref']) ";
 					$sSQL .= " union ";
 					$sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,";
-					$sSQL .= "get_tiger_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
+					$sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
 					$sSQL .= "null as placename,";
 					$sSQL .= "null as ref,";
 					$sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
@@ -854,7 +854,17 @@
 					$sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) ";
 					$sSQL .= "group by place_id";
 					if (!$bDeDupe) $sSQL .= ",place_id";
-					$sSQL .= ",get_tiger_address_by_language(place_id, $sLanguagePrefArraySQL) ";
+					$sSQL .= " union ";
+					$sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,";
+					$sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
+					$sSQL .= "null as placename,";
+					$sSQL .= "null as ref,";
+					$sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
+					$sSQL .= $sOrderSQL." as porder ";
+					$sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
+					$sSQL .= "group by place_id";
+					if (!$bDeDupe) $sSQL .= ",place_id";
+					$sSQL .= ",get_address_by_language(place_id, $sLanguagePrefArraySQL) ";
 					$sSQL .= "order by rank_search,rank_address,porder asc";
 					if (CONST_Debug) var_dump('<hr>',$sSQL);
 					$aSearchResults = $oDB->getAll($sSQL);