tigger data import

lib/lib.php

@@ -24,7 +24,14 @@
                 $aLoadAverage = explode(' ',$sLoadAverage);
 		return (int)$aLoadAverage[0];
 	}
-	
+
+	function getProcessorCount()
+	{
+		$sCPU = file_get_contents('/proc/cpuinfo');
+		preg_match_all('#processor	: [0-9]+#', $sCPU, $aMatches);
+		return sizeof($aMatches[0]);
+	}
+
 	function bySearchRank($a, $b)
 	{
 		if ($a['iSearchRank'] == $b['iSearchRank']) return 0;
@@ -49,7 +56,7 @@
 		{
 			$_SERVER["HTTP_ACCEPT_LANGUAGE"] = $_GET['accept-language'];
 		}
-		
+
 		$aLanguages = array();
 		if (preg_match_all('/(([a-z]{1,8})(-[a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i', $_SERVER['HTTP_ACCEPT_LANGUAGE'], $aLanguagesParse, PREG_SET_ORDER))
 		{

lib/template/search-html.php

@@ -8,8 +8,8 @@
 	<base href="<?php echo CONST_Website_BaseURL;?>" />
 	<link href="nominatim.xml" rel="search" title="Nominatim Search" type="application/opensearchdescription+xml" />
 
-	<script src="OpenLayers.js"></script>
-	<script src="http://www.openstreetmap.org/openlayers/OpenStreetMap.js"></script>
+	<script src="js/OpenLayers.js"></script>
+	<script src="js/OpenStreetMap.js"></script>
 	<script src="prototype-1.6.0.3.js"></script>
 
 	<style>

settings/settings.php

@@ -5,7 +5,7 @@
 
 	// General settings
 	@define('CONST_Debug', false);
-	@define('CONST_Database_DSN', 'pgsql://@/nominatim');
+	@define('CONST_Database_DSN', 'pgsql://brian@/nominatim');
 
 	// Paths
 	@define('CONST_Path_Postgresql_Contrib', '/usr/share/postgresql/9.0/contrib');

sql/functions.sql

@@ -2205,4 +2205,48 @@ CREATE AGGREGATE array_agg(INT[])
     initcond = '{}'
 );
 
+CREATE OR REPLACE FUNCTION tigger_create_interpolation(linegeo GEOMETRY, in_startnumber INTEGER, 
+  in_endnumber INTEGER, interpolationtype TEXT, 
+  in_street TEXT, in_isin TEXT, in_postcode TEXT) RETURNS INTEGER
+  AS $$
+DECLARE
+  
+  startnumber INTEGER;
+  stepsize INTEGER;
+  housenum INTEGER;
+  newpoints INTEGER;
+  numberrange INTEGER;
 
+BEGIN
+
+  numberrange := in_endnumber - in_startnumber;
+
+  IF (interpolationtype = 'odd' AND startnumber%2 = 0) OR (interpolationtype = 'even' AND startnumber%2 = 1) THEN
+    startnumber := in_startnumber + 1;
+    stepsize := 2;
+  ELSE
+    IF (interpolationtype = 'odd' OR interpolationtype = 'even') THEN
+      startnumber := in_startnumber;
+      stepsize := 2;
+    ELSE -- everything else assumed to be 'all'
+      startnumber := in_startnumber;
+      stepsize := 1;
+    END IF;
+  END IF;
+
+--this is a one time operation - skip the delete
+--delete from placex where osm_type = 'N' and osm_id = prevnode.osm_id and type = 'house' and place_id != prevnode.place_id;
+
+  newpoints := 0;
+  FOR housenum IN startnumber..in_endnumber BY stepsize LOOP
+    insert into placex (osm_type, osm_id, class, type, admin_level, housenumber, street, isin, postcode, 
+      country_code, parent_place_id, rank_address, rank_search, indexed_status, geometry)
+    values ('T', nextval('seq_tigger_house'), 'place', 'house', null, housenum, in_street, in_isin, in_postcode,
+      'us', null, 30, 30, 1, ST_Line_Interpolate_Point(linegeo, (housenum::float-in_startnumber::float)/numberrange::float));
+    newpoints := newpoints + 1;
+  END LOOP;
+
+  RETURN newpoints;
+END;
+$$
+LANGUAGE plpgsql;

utils/setup.php

@@ -10,11 +10,14 @@
 		array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
 		array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
 
-		array('all', '', 0, 1, 1, 1, 'realpath', 'Do the complete process'),
+		array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
+		array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
+
+		array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
 
 		array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
 		array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
-		array('import-data', '', 0, 1, 1, 1, 'realpath', 'Import a osm file'),
+		array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
 		array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
 		array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
 		array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
@@ -24,7 +27,7 @@
 
 	$bDidSomething = false;
 
-	if ($aCMDResult['create-db'] || isset($aCMDResult['all']))
+	if ($aCMDResult['create-db'] || $aCMDResult['all'])
 	{
 		$bDidSomething = true;
 		$oDB =& DB::connect(CONST_Database_DSN, false);
@@ -35,7 +38,7 @@
 		passthru('createdb nominatim');
 	}
 
-	if ($aCMDResult['create-db'] || isset($aCMDResult['all']))
+	if ($aCMDResult['create-db'] || $aCMDResult['all'])
 	{
 		$bDidSomething = true;
 		// TODO: path detection, detection memory, etc.
@@ -55,22 +58,22 @@
 		pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
 	}
 
-	if (isset($aCMDResult['all']) && !isset($aCMDResult['import-data'])) $aCMDResult['import-data'] = $aCMDResult['all'];
-	if (isset($aCMDResult['import-data']) && $aCMDResult['import-data'])
+	if ($aCMDResult['import-data'] || $aCMDResult['all'])
 	{
 		$bDidSomething = true;
-		passthru(CONST_BasePath.'/osm2pgsql/osm2pgsql -lsc -O gazetteer -C 10000 --hstore -d nominatim '.$aCMDResult['import-data']);
+		passthru(CONST_BasePath.'/osm2pgsql/osm2pgsql -lsc -O gazetteer -C 10000 --hstore -d nominatim '.$aCMDResult['osm-file']);
 	}
 
-	if ($aCMDResult['create-functions'] || isset($aCMDResult['all']))
+	if ($aCMDResult['create-functions'] || $aCMDResult['all'])
 	{
 		$bDidSomething = true;
+		if (!file_exists(CONST_BasePath.'/module/nominatim.so')) fail("nominatim module not built");
 		$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 		$sTemplate = str_replace('{modulepath}',CONST_BasePath.'/module', $sTemplate);
 		pgsqlRunScript($sTemplate);
 	}
 
-	if ($aCMDResult['create-tables'] || isset($aCMDResult['all']))
+	if ($aCMDResult['create-tables'] || $aCMDResult['all'])
 	{
 		$bDidSomething = true;
 		pgsqlRunScriptFile(CONST_BasePath.'/sql/tables.sql');
@@ -81,7 +84,7 @@
 		pgsqlRunScript($sTemplate);
 	}
 
-	if ($aCMDResult['create-partitions'] || isset($aCMDResult['all']))
+	if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
 	{
 		$bDidSomething = true;
 		$oDB =& getDB();
@@ -107,7 +110,7 @@
 		pgsqlRunScript($sTemplate);
 	}
 
-	if ($aCMDResult['load-data'] || isset($aCMDResult['all']))
+	if ($aCMDResult['load-data'] || $aCMDResult['all'])
 	{
 		$bDidSomething = true;
 
@@ -131,7 +134,19 @@
 		if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 		echo '.';
 
-		$iInstances = 16;
+		// This is a pretty hard core defult - the number of processors in the box - 1
+		$iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
+		if ($iInstances < 1)
+		{
+			$iInstances = 1;
+			echo "WARNING: reseting threads to $iInstances\n";
+		}
+		if ($iInstances > getProcessorCount())
+		{
+			$iInstances = getProcessorCount();
+			echo "WARNING: reseting threads to $iInstances\n";
+		}
+
 		$aDBInstances = array();
 		for($i = 0; $i < $iInstances; $i++)
 		{
@@ -139,7 +154,7 @@
 			$sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
 			$sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
 			$sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
-var_dump($sSQL);
+			if ($aCMDResult['verbose']) echo "$sSQL\n";
 			if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 		}
 		$bAnyBusy = true;

website/search.php

@@ -17,6 +17,7 @@
 	$bDeDupe = isset($_GET['dedupe'])?(bool)$_GET['dedupe']:true;
 	$bReverseInPlan = false;
 	$iLimit = isset($_GET['limit'])?(int)$_GET['limit']:10;
+	$iOffset = isset($_GET['offset'])?(int)$_GET['offset']:0;
 	$iMaxRank = 20;
 	if ($iLimit > 100) $iLimit = 100;
 
@@ -1024,4 +1025,4 @@
 
 	if (CONST_Debug) exit;
 
-	include('.htlib/output/search-'.$sOutputFormat.'.php');
+	include(CONST_BasePath.'/lib/template/search-'.$sOutputFormat.'.php');