From 1028c2444cd5789860de283cc77eae914daae503 Mon Sep 17 00:00:00 2001 From: Brian Quinion Date: Wed, 6 Apr 2011 18:19:40 +0000 Subject: [PATCH] move partiondef config file to settings, add --create-website option to build symlinks automatically --- {nominatim => settings}/partitionedtags.def | 0 sql/tables-minimal.sql | 155 ++++++++++++++++++++ utils/setup.php | 64 +++++++- 3 files changed, 211 insertions(+), 8 deletions(-) rename {nominatim => settings}/partitionedtags.def (100%) create mode 100644 sql/tables-minimal.sql diff --git a/nominatim/partitionedtags.def b/settings/partitionedtags.def similarity index 100% rename from nominatim/partitionedtags.def rename to settings/partitionedtags.def diff --git a/sql/tables-minimal.sql b/sql/tables-minimal.sql new file mode 100644 index 00000000..eb529252 --- /dev/null +++ b/sql/tables-minimal.sql @@ -0,0 +1,155 @@ +drop table import_npi_log; +CREATE TABLE import_npi_log ( + npiid integer, + batchend timestamp, + batchsize integer, + starttime timestamp, + endtime timestamp, + event text + ); + +drop table IF EXISTS word; +CREATE TABLE word ( + word_id INTEGER, + word_token text, + word_trigram text, + word text, + class text, + type text, + country_code varchar(2), + search_name_count INTEGER, + operator TEXT + ); +SELECT AddGeometryColumn('word', 'location', 4326, 'GEOMETRY', 2); +CREATE INDEX idx_word_word_id on word USING BTREE (word_id); +CREATE INDEX idx_word_word_token on word USING BTREE (word_token); +GRANT SELECT ON word TO "www-data" ; +DROP SEQUENCE seq_word; +CREATE SEQUENCE seq_word start 1; + +drop table IF EXISTS location_property CASCADE; +CREATE TABLE location_property ( + place_id INTEGER, + partition integer, + parent_place_id INTEGER, + housenumber TEXT, + postcode TEXT + ); +SELECT AddGeometryColumn('location_property', 'centroid', 4326, 'POINT', 2); + +CREATE TABLE location_property_aux () INHERITS (location_property); +CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id); +CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id); +CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber); + +CREATE TABLE location_property_tiger () INHERITS (location_property); +CREATE INDEX idx_location_property_tiger_place_id ON location_property_tiger USING BTREE (place_id); +CREATE INDEX idx_location_property_tiger_parent_place_id ON location_property_tiger USING BTREE (parent_place_id); +CREATE INDEX idx_location_property_tiger_housenumber_parent_place_id ON location_property_tiger USING BTREE (parent_place_id, housenumber); + +drop table IF EXISTS search_name_blank CASCADE; +CREATE TABLE search_name_blank ( + place_id INTEGER, + search_rank integer, + address_rank integer, + importance FLOAT, + country_code varchar(2), + name_vector integer[], + nameaddress_vector integer[] + ); +SELECT AddGeometryColumn('search_name_blank', 'centroid', 4326, 'GEOMETRY', 2); + +drop table IF EXISTS search_name; +CREATE TABLE search_name () INHERITS (search_name_blank); +CREATE INDEX search_name_name_vector_idx ON search_name USING GIN (name_vector gin__int_ops) WITH (fastupdate = off); +CREATE INDEX searchnameplacesearch_search_nameaddress_vector_idx ON search_name USING GIN (nameaddress_vector gin__int_ops) WITH (fastupdate = off); +CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid); +CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id); + +drop table IF EXISTS place_addressline; +CREATE TABLE place_addressline ( + place_id INTEGER, + address_place_id INTEGER, + fromarea boolean, + isaddress boolean, + distance float, + cached_rank_address integer + ); +CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id); +CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id); + +drop table IF EXISTS place_boundingbox CASCADE; +CREATE TABLE place_boundingbox ( + place_id INTEGER, + minlat float, + maxlat float, + minlon float, + maxlon float, + numfeatures integer, + area float + ); +CREATE INDEX idx_place_boundingbox_place_id on place_boundingbox USING BTREE (place_id); +SELECT AddGeometryColumn('place_boundingbox', 'outline', 4326, 'GEOMETRY', 2); +CREATE INDEX idx_place_boundingbox_outline ON place_boundingbox USING GIST (outline); +GRANT SELECT on place_boundingbox to "www-data" ; +GRANT INSERT on place_boundingbox to "www-data" ; + +drop table country; +CREATE TABLE country ( + country_code varchar(2), + country_name hstore, + country_default_language_code varchar(2) + ); +SELECT AddGeometryColumn('country', 'geometry', 4326, 'POLYGON', 2); +insert into country select iso3166::varchar(2), 'name:en'->cntry_name, null, + ST_Transform(geometryn(the_geom, generate_series(1, numgeometries(the_geom))), 4326) from worldboundaries; +CREATE INDEX idx_country_country_code ON country USING BTREE (country_code); +CREATE INDEX idx_country_geometry ON country USING GIST (geometry); + +drop table placex; +CREATE TABLE placex ( + place_id INTEGER NOT NULL, + partition integer, + osm_type char(1), + osm_id INTEGER, + class TEXT NOT NULL, + type TEXT NOT NULL, + name HSTORE, + admin_level INTEGER, + housenumber TEXT, + street TEXT, + isin TEXT, + postcode TEXT, + country_code varchar(2), + extratags HSTORE, + parent_place_id INTEGER, + linked_place_id INTEGER, + rank_address INTEGER, + rank_search INTEGER, + importance FLOAT, + indexed_status INTEGER, + indexed_date TIMESTAMP, + geometry_sector INTEGER + ); +SELECT AddGeometryColumn('placex', 'geometry', 4326, 'GEOMETRY', 2); +CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id); +CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id); +CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search); +CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address); +CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry); +CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) where parent_place_id IS NOT NULL; + +DROP SEQUENCE seq_place; +CREATE SEQUENCE seq_place start 1; +GRANT SELECT on placex to "www-data" ; +GRANT UPDATE ON placex to "www-data" ; +GRANT SELECT ON search_name to "www-data" ; +GRANT DELETE on search_name to "www-data" ; +GRANT INSERT on search_name to "www-data" ; +GRANT SELECT on place_addressline to "www-data" ; +GRANT INSERT ON place_addressline to "www-data" ; +GRANT DELETE on place_addressline to "www-data" ; +GRANT SELECT ON seq_word to "www-data" ; +GRANT UPDATE ON seq_word to "www-data" ; +GRANT INSERT ON word to "www-data" ; +GRANT SELECT on country to "www-data" ; diff --git a/utils/setup.php b/utils/setup.php index 0b5392da..5a54ef77 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -19,6 +19,7 @@ array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), + array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'), array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'), array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), @@ -29,6 +30,7 @@ array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'), array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'), + array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'), ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); @@ -56,6 +58,8 @@ $aCMDResult['osmosis-init-date'] = date('Y-m-d', $iTime).'T22:00:00Z'; } } + $aDSNInfo = DB::parseDSN(CONST_Database_DSN); + if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; if ($aCMDResult['create-db'] || $aCMDResult['all']) { @@ -64,9 +68,9 @@ $oDB =& DB::connect(CONST_Database_DSN, false); if (!PEAR::isError($oDB)) { - fail('database already exists'); + fail('database already exists ('.CONST_Database_DSN.')'); } - passthru('createdb nominatim'); + passthru('createdb '.$aDSNInfo['database']); } if ($aCMDResult['create-db'] || $aCMDResult['all']) @@ -76,7 +80,7 @@ // TODO: path detection, detection memory, etc. $oDB =& getDB(); - passthru('createlang plpgsql nominatim'); + passthru('createlang plpgsql '.$aDSNInfo['database']); pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/_int.sql'); pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql'); pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql'); @@ -96,10 +100,10 @@ echo "Import\n"; $bDidSomething = true; - $osm2pgsql = CONST_BasePath.'/osm2pgsql/osm2pgsql'; - if (!file_exists($osm2pgsql)) $osm2pgsql = trim(`which osm2pgsql`); - if (!file_exists($osm2pgsql)) fail("please download and build osm2pgsql"); - passthru($osm2pgsql.' -lsc -O gazetteer -C 10000 --hstore -d nominatim '.$aCMDResult['osm-file']); + $osm2pgsql = CONST_BasePath.'/osm2pgsql/osm2pgsql'; + if (!file_exists($osm2pgsql)) $osm2pgsql = trim(`which osm2pgsql`); + if (!file_exists($osm2pgsql)) fail("please download and build osm2pgsql"); + passthru($osm2pgsql.' -lsc -O gazetteer -C 10000 --hstore -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']); $oDB =& getDB(); $x = $oDB->getRow('select * from place limit 1'); @@ -116,6 +120,36 @@ pgsqlRunScript($sTemplate); } + if ($aCMDResult['create-minimal-tables']) + { + echo "Minimal Tables\n"; + $bDidSomething = true; + pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql'); + + $sScript = ''; + + // Backstop the import process - easliest possible import id + $sScript .= "insert into import_npi_log values (18022);\n"; + + $hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r"); + if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def'); + + while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#') + { + list($sClass, $sType) = explode(' ', trim($sLine)); + $sScript .= "create table place_classtype_".$sClass."_".$sType." as "; + $sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n"; + + $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid "; + $sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n"; + + $sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id "; + $sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n"; + } + fclose($hFile); + pgsqlRunScript($sScript); + } + if ($aCMDResult['create-tables'] || $aCMDResult['all']) { echo "Tables\n"; @@ -327,7 +361,21 @@ $bDidSomething = true; $sOutputFile = ''; if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output']; - passthru(CONST_BasePath.'/nominatim/nominatim -i -d nominatim -t '.$iInstances.$sOutputFile); + passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile); + } + + if (isset($aCMDResult['create-website'])) + { + $bDidSomething = true; + $sTargetDir = $aCMDResult['create-website']; + if (!is_dir($sTargetDir)) fail('please specify a directory to setup'); + @symlink(CONST_BasePath.'/website/details.php', $sTargetDir.'/details.php'); + @symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php'); + @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php'); + @symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php'); + @symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images'); + @symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js'); + echo "Symlinks created\n"; } if (!$bDidSomething)