Merge pull request #2164 from lonvia/add-make-install

Make Nominatim installable

.github/actions/build-nominatim/action.yml

@@ -9,21 +9,21 @@ runs:
             sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium python3-dotenv
           shell: bash
 
+        - name: Download dependencies
+          run: |
+              if [ ! -f country_grid.sql.gz ]; then
+                  wget --no-verbose https://www.nominatim.org/data/country_grid.sql.gz
+              fi
+              cp country_grid.sql.gz Nominatim/data/country_osm_grid.sql.gz
+          shell: bash
+
         - name: Configure
-          run: mkdir build && cd build && cmake ..
+          run: mkdir build && cd build && cmake ../Nominatim
           shell: bash
 
         - name: Build
           run: |
               make -j2 all
-              ./nominatim refresh --website
+              sudo make install
           shell: bash
           working-directory: build
-
-        - name: Download dependencies
-          run: |
-              if [ ! -f data/country_osm_grid.sql.gz ]; then
-                  wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
-              fi
-          shell: bash
-

.github/workflows/ci-tests.yml

@@ -19,6 +19,7 @@ jobs:
             - uses: actions/checkout@v2
               with:
                   submodules: true
+                  path: Nominatim
 
             - name: Setup PHP
               uses: shivammathur/setup-php@v2
@@ -35,35 +36,37 @@ jobs:
             - uses: actions/cache@v2
               with:
                   path: |
-                     data/country_osm_grid.sql.gz
+                     country_grid.sql.gz
-                     monaco-latest.osm.pbf
+                  key: nominatim-country-data-${{ steps.get-date.outputs.date }}
-                  key: nominatim-data-${{ steps.get-date.outputs.date }}
 
-            - uses: ./.github/actions/setup-postgresql
+            - uses: ./Nominatim/.github/actions/setup-postgresql
               with:
                   postgresql-version: ${{ matrix.postgresql }}
                   postgis-version: ${{ matrix.postgis }}
-            - uses: ./.github/actions/build-nominatim
+            - uses: ./Nominatim/.github/actions/build-nominatim
 
             - name: Install test prerequsites
               run: sudo apt-get install -y -qq php-codesniffer pylint python3-pytest python3-behave
 
             - name: PHP linting
               run: phpcs --report-width=120 .
+              working-directory: Nominatim
 
             - name: Python linting
               run: pylint --extension-pkg-whitelist=osmium nominatim
+              working-directory: Nominatim
 
             - name: PHP unit tests
               run: phpunit ./
-              working-directory: test/php
+              working-directory: Nominatim/test/php
 
             - name: Python unit tests
               run: py.test-3 test/python
+              working-directory: Nominatim
 
             - name: BDD tests
-              run: behave -DREMOVE_TEMPLATE=1 --format=progress3
+              run: behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
-              working-directory: test/bdd
+              working-directory: Nominatim/test/bdd
 
     import:
         runs-on: ubuntu-20.04
@@ -72,6 +75,7 @@ jobs:
             - uses: actions/checkout@v2
               with:
                   submodules: true
+                  path: Nominatim
 
             - name: Get Date
               id: get-date
@@ -82,46 +86,55 @@ jobs:
             - uses: actions/cache@v2
               with:
                   path: |
-                     data/country_osm_grid.sql.gz
+                     country_grid.sql.gz
-                     monaco-latest.osm.pbf
+                  key: nominatim-country-data-${{ steps.get-date.outputs.date }}
-                  key: nominatim-data-${{ steps.get-date.outputs.date }}
 
-            - uses: ./.github/actions/setup-postgresql
+            - uses: actions/cache@v2
+              with:
+                  path: |
+                     monaco-latest.osm.pbf
+                  key: nominatim-test-data-${{ steps.get-date.outputs.date }}
+
+            - uses: ./Nominatim/.github/actions/setup-postgresql
               with:
                   postgresql-version: 13
                   postgis-version: 3
-            - uses: ./.github/actions/build-nominatim
+            - uses: ./Nominatim/.github/actions/build-nominatim
 
-            - name: Download import data
+            - name: Clean installation
+              run: rm -rf Nominatim build
+              shell: bash
+
+            - name: Prepare import environment
               run: |
                   if [ ! -f monaco-latest.osm.pbf ]; then
                       wget --no-verbose https://download.geofabrik.de/europe/monaco-latest.osm.pbf
                   fi
+                  mkdir data-env
+                  cd data-env
               shell: bash
 
             - name: Import
-              run: |
+              run: nominatim import --osm-file ../monaco-latest.osm.pbf
-                  mkdir data-env
-                  cd data-env
-                  ../build/nominatim import --osm-file ../monaco-latest.osm.pbf
               shell: bash
+              working-directory: data-env
 
             - name: Import special phrases
-              run: ../build/nominatim special-phrases --from-wiki | psql -d nominatim
+              run: nominatim special-phrases --from-wiki | psql -d nominatim
               working-directory: data-env
 
             - name: Check import
-              run: ../build/nominatim check-database
+              run: nominatim admin --check-database
               working-directory: data-env
 
             - name: Run update
               run: |
-                   ../build/nominatim replication --init
+                   nominatim replication --init
-                   ../build/nominatim replication --once
+                   nominatim replication --once
               working-directory: data-env
 
             - name: Run reverse-only import
-              run : |
+              run : nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only
-                  echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' > .env
-                  ../build/nominatim import --osm-file ../monaco-latest.osm.pbf --reverse-only
               working-directory: data-env
+              env:
+                  NOMINATIM_DATABASE_DSN: pgsql:dbname=reverse

CMakeLists.txt

@@ -97,6 +97,17 @@ endif()
 #-----------------------------------------------------------------------------
 
 if (BUILD_IMPORTER)
+   find_file(COUNTRY_GRID_FILE country_osm_grid.sql.gz
+             PATHS ${PROJECT_SOURCE_DIR}/data
+             NO_DEFAULT_PATH
+             DOC "Location of the country grid file."
+            )
+
+   if (NOT COUNTRY_GRID_FILE)
+       message(FATAL_ERROR "\nYou need to download the country_osm_grid first:\n"
+                           "    wget -O ${PROJECT_SOURCE_DIR}/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz")
+   endif()
+
    set(CUSTOMSCRIPTS
        check_import_finished.php
        country_languages.php
@@ -197,3 +208,54 @@ endif()
 if (BUILD_DOCS)
    add_subdirectory(docs)
 endif()
+
+#-----------------------------------------------------------------------------
+# Installation
+#-----------------------------------------------------------------------------
+
+
+include(GNUInstallDirs)
+set(NOMINATIM_DATADIR ${CMAKE_INSTALL_FULL_DATADIR}/${PROJECT_NAME})
+set(NOMINATIM_LIBDIR ${CMAKE_INSTALL_FULL_LIBDIR}/${PROJECT_NAME})
+set(NOMINATIM_CONFIGDIR ${CMAKE_INSTALL_FULL_SYSCONFDIR}/${PROJECT_NAME})
+
+if (BUILD_IMPORTER)
+    configure_file(${PROJECT_SOURCE_DIR}/cmake/tool-installed.tmpl installed.bin)
+    install(PROGRAMS ${PROJECT_BINARY_DIR}/installed.bin
+            DESTINATION ${CMAKE_INSTALL_BINDIR}
+            RENAME nominatim)
+
+    install(DIRECTORY nominatim
+            DESTINATION ${NOMINATIM_LIBDIR}/lib-python
+            FILES_MATCHING PATTERN "*.py"
+            PATTERN __pycache__ EXCLUDE)
+    install(DIRECTORY lib-sql DESTINATION ${NOMINATIM_LIBDIR})
+
+    install(FILES data/country_name.sql
+                  ${COUNTRY_GRID_FILE}
+                  data/words.sql
+            DESTINATION ${NOMINATIM_DATADIR})
+endif()
+
+if (BUILD_OSM2PGSQL)
+    install(TARGETS osm2pgsql RUNTIME DESTINATION ${NOMINATIM_LIBDIR})
+endif()
+
+if (BUILD_MODULE)
+    install(PROGRAMS ${PROJECT_BINARY_DIR}/module/nominatim.so
+            DESTINATION ${NOMINATIM_LIBDIR}/module)
+endif()
+
+if (BUILD_API)
+    install(DIRECTORY lib-php DESTINATION ${NOMINATIM_LIBDIR})
+endif()
+
+install(FILES settings/env.defaults
+              settings/address-levels.json
+              settings/phrase_settings.php
+              settings/import-admin.style
+              settings/import-street.style
+              settings/import-address.style
+              settings/import-full.style
+              settings/import-extratags.style
+        DESTINATION ${NOMINATIM_CONFIGDIR})

README.md

@@ -41,12 +41,13 @@ A quick summary of the necessary steps:
         cd build
         cmake ..
         make
+        sudo make install
 
 2. Create a project directory, get OSM data and import:
 
         mkdir nominatim-project
         cd nominatim-project
-        ~/build/nominatim import --osm-file <your planet file>
+        nominatim import --osm-file <your planet file>
 
 3. Point your webserver to the nominatim-project/website directory.
 

cmake/script.tmpl

@@ -1,13 +1,14 @@
 #!@PHP_BIN@ -Cq
 <?php
-require('@CMAKE_SOURCE_DIR@/lib/dotenv_loader.php');
+require('@CMAKE_SOURCE_DIR@/lib-php/dotenv_loader.php');
 
 @define('CONST_Default_ModulePath', '@CMAKE_BINARY_DIR@/module');
 @define('CONST_Default_Osm2pgsql', '@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql');
-@define('CONST_BinDir', '@CMAKE_SOURCE_DIR@/utils');
+@define('CONST_DataDir', '@CMAKE_SOURCE_DIR@/data');
-@define('CONST_DataDir', '@CMAKE_SOURCE_DIR@');
+@define('CONST_SqlDir', '@CMAKE_SOURCE_DIR@/lib-sql');
+@define('CONST_ConfigDir', '@CMAKE_SOURCE_DIR@/settings');
 
 loadDotEnv();
 $_SERVER['NOMINATIM_NOMINATIM_TOOL'] = '@CMAKE_BINARY_DIR@/nominatim';
 
-require_once('@CMAKE_SOURCE_DIR@/lib/admin/@script_source@');
+require_once('@CMAKE_SOURCE_DIR@/lib-php/admin/@script_source@');

cmake/tool-installed.tmpl

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+import sys
+import os
+
+sys.path.insert(1, '@NOMINATIM_LIBDIR@/lib-python')
+
+os.environ['NOMINATIM_NOMINATIM_TOOL'] = os.path.abspath(__file__)
+
+from nominatim import cli
+
+exit(cli.nominatim(module_dir='@NOMINATIM_LIBDIR@/module',
+                   osm2pgsql_path='@NOMINATIM_LIBDIR@/osm2pgsql',
+                   phplib_dir='@NOMINATIM_LIBDIR@/lib-php',
+                   sqllib_dir='@NOMINATIM_LIBDIR@/lib-sql',
+                   data_dir='@NOMINATIM_DATADIR@',
+                   config_dir='@NOMINATIM_CONFIGDIR@',
+                   phpcgi_path='@PHPCGI_BIN@'))

cmake/tool.tmpl

@@ -10,6 +10,8 @@ from nominatim import cli
 
 exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module',
                    osm2pgsql_path='@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql',
-                   phplib_dir='@CMAKE_SOURCE_DIR@/lib',
+                   phplib_dir='@CMAKE_SOURCE_DIR@/lib-php',
-                   data_dir='@CMAKE_SOURCE_DIR@',
+                   sqllib_dir='@CMAKE_SOURCE_DIR@/lib-sql',
+                   data_dir='@CMAKE_SOURCE_DIR@/data',
+                   config_dir='@CMAKE_SOURCE_DIR@/settings',
                    phpcgi_path='@PHPCGI_BIN@'))

data/gb_postcode_table.sql

@@ -1,26 +0,0 @@
--- This data contains Ordnance Survey data © Crown copyright and database right 2010. 
--- Code-Point Open contains Royal Mail data © Royal Mail copyright and database right 2010.
--- OS data may be used under the terms of the OS OpenData licence:
--- http://www.ordnancesurvey.co.uk/oswebsite/opendata/licence/docs/licence.pdf
-
-SET statement_timeout = 0;
-SET client_encoding = 'UTF8';
-SET standard_conforming_strings = off;
-SET check_function_bodies = false;
-SET client_min_messages = warning;
-SET escape_string_warning = off;
-
-SET search_path = public, pg_catalog;
-
-SET default_tablespace = '';
-
-SET default_with_oids = false;
-
-CREATE TABLE gb_postcode (
-    id integer,
-    postcode character varying(9),
-    geometry geometry,
-    CONSTRAINT enforce_dims_geometry CHECK ((st_ndims(geometry) = 2)),
-    CONSTRAINT enforce_srid_geometry CHECK ((st_srid(geometry) = 4326))
-);
-

data/us_postcode_table.sql

@@ -1,16 +0,0 @@
-SET statement_timeout = 0;
-SET client_encoding = 'UTF8';
-SET check_function_bodies = false;
-SET client_min_messages = warning;
-
-SET search_path = public, pg_catalog;
-
-SET default_tablespace = '';
-
-SET default_with_oids = false;
-
-CREATE TABLE us_postcode (
-    postcode text,
-    x double precision,
-    y double precision
-);

docs/admin/Advanced-Installations.md

@@ -155,7 +155,7 @@ Make sure that the PostgreSQL server package is installed on the machine
 the PostgreSQL server itself.
 
 Download and compile Nominatim as per standard instructions. Once done, you find
-the nomrmalization library in `build/module/nominatim.so`. Copy the file to
+the normalization library in `build/module/nominatim.so`. Copy the file to
 the database server at a location where it is readable and executable by the
 PostgreSQL server process.
 

docs/admin/Import.md

@@ -2,7 +2,8 @@
 
 The following instructions explain how to create a Nominatim database
 from an OSM planet file. It is assumed that you have already successfully
-installed the Nominatim software itself. If this is not the case, return to the
+installed the Nominatim software itself and the `nominatim` tool can be found
+in your `PATH`. If this is not the case, return to the
 [installation page](Installation.md).
 
 ## Creating the project directory
@@ -10,10 +11,11 @@ installed the Nominatim software itself. If this is not the case, return to the
 Before you start the import, you should create a project directory for your
 new database installation. This directory receives all data that is related
 to a single Nominatim setup: configuration, extra data, etc. Create a project
-directory apart from the Nominatim software:
+directory apart from the Nominatim software and change into the directory:
 
 ```
 mkdir ~/nominatim-planet
+cd ~/nominatim-planet
 ```
 
 In the following, we refer to the project directory as `$PROJECT_DIR`. To be
@@ -25,18 +27,8 @@ export PROJECT_DIR=~/nominatim-planet
 
 The Nominatim tool assumes per default that the current working directory is
 the project directory but you may explicitly state a different directory using
-the `--project-dir` parameter. The following instructions assume that you have
+the `--project-dir` parameter. The following instructions assume that you run
-added the Nominatim build directory to your PATH and run all directories from
+all commands from the project directory.
-the project directory. If you haven't done yet, add the build directory to your
-path and change to the new project directory:
-
-```
-export PATH=~/Nominatim/build:$PATH
-cd $PROJECT_DIR
-```
-
-Of course, you have to replace the path above with the location of your build
-directory.
 
 !!! tip "Migration Tip"
 
@@ -242,7 +234,7 @@ reduce the cache size or even consider using a flatnode file.
 Run this script to verify all required tables and indices got created successfully.
 
 ```sh
-nominatim check-database
+nominatim admin --check-database
 ```
 
 Now you can try out your installation by running:

docs/admin/Installation.md

@@ -40,14 +40,15 @@ For running Nominatim:
   * [PostGIS](https://postgis.net) (2.2+)
   * [Python 3](https://www.python.org/) (3.5+)
   * [Psycopg2](https://www.psycopg.org)
+  * [Python Dotenv](https://github.com/theskumar/python-dotenv)
   * [PHP](https://php.net) (7.0 or later)
   * PHP-pgsql
   * PHP-intl (bundled with PHP)
-  * [Python Dotenv](https://github.com/theskumar/python-dotenv)
+  ( PHP-cgi (for running queries from the command line)
 
 For running continuous updates:
 
-  * [pyosmium](https://osmcode.org/pyosmium/) (with Python 3)
+  * [pyosmium](https://osmcode.org/pyosmium/)
 
 For dependencies for running tests and building documentation, see
 the [Development section](../develop/Development-Environment.md).
@@ -143,6 +144,16 @@ build at the same level as the Nominatim source directory run:
 ```
 cmake ../Nominatim
 make
+sudo make install
+```
+
+Nominatim installs itself into `/usr/local` per default. To choose a different
+installation directory add `-DCMAKE_INSTALL_PREFIX=<install root>` to the
+cmake command. Make sure that the `bin` directory is available in your path
+in that case, e.g.
+
+```
+export PATH=<install root>/bin:$PATH
 ```
 
 Now continue with [importing the database](Import.md).

docs/admin/Migration.md

@@ -37,8 +37,8 @@ functionality of each script:
 * ./utils/setup.php: `import`, `freeze`, `refresh`
 * ./utils/update.php: `replication`, `add-data`, `index`, `refresh`
 * ./utils/specialphrases.php: `special-phrases`
-* ./utils/check_import_finished.php: `check-database`
+* ./utils/check_import_finished.php: `admin`
-* ./utils/warm.php: `warm`
+* ./utils/warm.php: `admin`
 * ./utils/export.php: `export`
 
 Try `nominatim <command> --help` for more information about each subcommand.

lib-php/dotenv_loader.php

@@ -5,7 +5,7 @@ require('Symfony/Component/Dotenv/autoload.php');
 function loadDotEnv()
 {
     $dotenv = new \Symfony\Component\Dotenv\Dotenv();
-    $dotenv->load(CONST_DataDir.'/settings/env.defaults');
+    $dotenv->load(CONST_ConfigDir.'/env.defaults');
 
     if (file_exists('.env')) {
         $dotenv->load('.env');

lib-php/lib.php

@@ -7,7 +7,8 @@ function loadSettings($sProjectDir)
     // the installed scripts. Neither setting is part of the official
     // set of settings.
     defined('CONST_DataDir') or define('CONST_DataDir', $_SERVER['NOMINATIM_DATADIR']);
-    defined('CONST_BinDir') or define('CONST_BinDir', $_SERVER['NOMINATIM_BINDIR']);
+    defined('CONST_SqlDir') or define('CONST_SqlDir', $_SERVER['NOMINATIM_SQLDIR']);
+    defined('CONST_ConfigDir') or define('CONST_ConfigDir', $_SERVER['NOMINATIM_CONFIGDIR']);
     defined('CONST_Default_ModulePath') or define('CONST_Default_ModulePath', $_SERVER['NOMINATIM_DATABASE_MODULE_SRC_PATH']);
 }
 
@@ -36,7 +37,7 @@ function getSettingConfig($sConfName, $sSystemConfig)
     $sValue = $_SERVER['NOMINATIM_'.$sConfName];
 
     if (!$sValue) {
-        return CONST_DataDir.'/settings/'.$sSystemConfig;
+        return CONST_ConfigDir.'/'.$sSystemConfig;
     }
 
     return $sValue;

lib-php/setup/SetupClass.php

@@ -166,29 +166,8 @@ class SetupFunctions
         // Try accessing the C module, so we know early if something is wrong
         $this->checkModulePresence(); // raises exception on failure
 
-        if (!file_exists(CONST_DataDir.'/data/country_osm_grid.sql.gz')) {
+        $this->pgsqlRunScriptFile(CONST_DataDir.'/country_name.sql');
-            echo 'Error: you need to download the country_osm_grid first:';
+        $this->pgsqlRunScriptFile(CONST_DataDir.'/country_osm_grid.sql.gz');
-            echo "\n    wget -O ".CONST_DataDir."/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n";
-            exit(1);
-        }
-        $this->pgsqlRunScriptFile(CONST_DataDir.'/data/country_name.sql');
-        $this->pgsqlRunScriptFile(CONST_DataDir.'/data/country_osm_grid.sql.gz');
-        $this->pgsqlRunScriptFile(CONST_DataDir.'/data/gb_postcode_table.sql');
-        $this->pgsqlRunScriptFile(CONST_DataDir.'/data/us_postcode_table.sql');
-
-        $sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz';
-        if (file_exists($sPostcodeFilename)) {
-            $this->pgsqlRunScriptFile($sPostcodeFilename);
-        } else {
-            warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
-        }
-
-        $sPostcodeFilename = CONST_InstallDir.'/us_postcode_data.sql.gz';
-        if (file_exists($sPostcodeFilename)) {
-            $this->pgsqlRunScriptFile($sPostcodeFilename);
-        } else {
-            warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
-        }
 
         if ($this->bNoPartitions) {
             $this->pgsqlRunScript('update country_name set partition = 0');
@@ -269,7 +248,7 @@ class SetupFunctions
     {
         info('Create Tables');
 
-        $sTemplate = file_get_contents(CONST_DataDir.'/sql/tables.sql');
+        $sTemplate = file_get_contents(CONST_SqlDir.'/tables.sql');
         $sTemplate = $this->replaceSqlPatterns($sTemplate);
 
         $this->pgsqlRunScript($sTemplate, false);
@@ -285,7 +264,7 @@ class SetupFunctions
     {
         info('Create Tables');
 
-        $sTemplate = file_get_contents(CONST_DataDir.'/sql/table-triggers.sql');
+        $sTemplate = file_get_contents(CONST_SqlDir.'/table-triggers.sql');
         $sTemplate = $this->replaceSqlPatterns($sTemplate);
 
         $this->pgsqlRunScript($sTemplate, false);
@@ -295,7 +274,7 @@ class SetupFunctions
     {
         info('Create Partition Tables');
 
-        $sTemplate = file_get_contents(CONST_DataDir.'/sql/partition-tables.src.sql');
+        $sTemplate = file_get_contents(CONST_SqlDir.'/partition-tables.src.sql');
         $sTemplate = $this->replaceSqlPatterns($sTemplate);
 
         $this->pgsqlRunPartitionScript($sTemplate);
@@ -366,7 +345,7 @@ class SetupFunctions
         // pre-create the word list
         if (!$bDisableTokenPrecalc) {
             info('Loading word list');
-            $this->pgsqlRunScriptFile(CONST_DataDir.'/data/words.sql');
+            $this->pgsqlRunScriptFile(CONST_DataDir.'/words.sql');
         }
 
         info('Load Data');
@@ -458,7 +437,7 @@ class SetupFunctions
             warn('Tiger data import selected but no files found in path '.$sTigerPath);
             return;
         }
-        $sTemplate = file_get_contents(CONST_DataDir.'/sql/tiger_import_start.sql');
+        $sTemplate = file_get_contents(CONST_SqlDir.'/tiger_import_start.sql');
         $sTemplate = $this->replaceSqlPatterns($sTemplate);
 
         $this->pgsqlRunScript($sTemplate, false);
@@ -512,7 +491,7 @@ class SetupFunctions
         }
 
         info('Creating indexes on Tiger data');
-        $sTemplate = file_get_contents(CONST_DataDir.'/sql/tiger_import_finish.sql');
+        $sTemplate = file_get_contents(CONST_SqlDir.'/tiger_import_finish.sql');
         $sTemplate = $this->replaceSqlPatterns($sTemplate);
 
         $this->pgsqlRunScript($sTemplate, false);
@@ -521,6 +500,23 @@ class SetupFunctions
     public function calculatePostcodes($bCMDResultAll)
     {
         info('Calculate Postcodes');
+        $this->pgsqlRunScriptFile(CONST_SqlDir.'/postcode_tables.sql');
+
+        $sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz';
+        if (file_exists($sPostcodeFilename)) {
+            $this->pgsqlRunScriptFile($sPostcodeFilename);
+        } else {
+            warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
+        }
+
+        $sPostcodeFilename = CONST_InstallDir.'/us_postcode_data.sql.gz';
+        if (file_exists($sPostcodeFilename)) {
+            $this->pgsqlRunScriptFile($sPostcodeFilename);
+        } else {
+            warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
+        }
+
+
         $this->db()->exec('TRUNCATE location_postcode');
 
         $sSQL  = 'INSERT INTO location_postcode';
@@ -620,12 +616,12 @@ class SetupFunctions
             $this->db()->exec("DROP INDEX $sIndexName;");
         }
 
-        $sTemplate = file_get_contents(CONST_DataDir.'/sql/indices.src.sql');
+        $sTemplate = file_get_contents(CONST_SqlDir.'/indices.src.sql');
         if (!$this->bDrop) {
-            $sTemplate .= file_get_contents(CONST_DataDir.'/sql/indices_updates.src.sql');
+            $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_updates.src.sql');
         }
         if (!$this->dbReverseOnly()) {
-            $sTemplate .= file_get_contents(CONST_DataDir.'/sql/indices_search.src.sql');
+            $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_search.src.sql');
         }
         $sTemplate = $this->replaceSqlPatterns($sTemplate);
 
@@ -736,8 +732,6 @@ class SetupFunctions
             fwrite($rFile, '@define(\'CONST_Debug\', $_GET[\'debug\'] ?? false);'."\n\n");
 
             fwriteConstDef($rFile, 'LibDir', CONST_LibDir);
-            fwriteConstDef($rFile, 'DataDir', CONST_DataDir);
-            fwriteConstDef($rFile, 'InstallDir', CONST_InstallDir);
             fwriteConstDef($rFile, 'Database_DSN', getSetting('DATABASE_DSN'));
             fwriteConstDef($rFile, 'Default_Language', getSetting('DEFAULT_LANGUAGE'));
             fwriteConstDef($rFile, 'Log_DB', getSettingBool('LOG_DB'));
@@ -753,8 +747,7 @@ class SetupFunctions
             fwriteConstDef($rFile, 'Use_US_Tiger_Data', getSettingBool('USE_US_TIGER_DATA'));
             fwriteConstDef($rFile, 'MapIcon_URL', getSetting('MAPICON_URL'));
 
-            // XXX scripts should go into the library.
+            fwrite($rFile, 'require_once(\''.CONST_LibDir.'/website/'.$sScript."');\n");
-            fwrite($rFile, 'require_once(\''.CONST_DataDir.'/website/'.$sScript."');\n");
             fclose($rFile);
 
             chmod(CONST_InstallDir.'/website/'.$sScript, 0755);

lib-php/setup_functions.php

@@ -27,7 +27,7 @@ function getImportStyle()
     $sStyle = getSetting('IMPORT_STYLE');
 
     if (in_array($sStyle, array('admin', 'street', 'address', 'full', 'extratags'))) {
-        return CONST_DataDir.'/settings/import-'.$sStyle.'.style';
+        return CONST_ConfigDir.'/import-'.$sStyle.'.style';
     }
 
     return $sStyle;

lib-sql/postcode_tables.sql

@@ -0,0 +1,15 @@
+DROP TABLE IF EXISTS gb_postcode;
+CREATE TABLE gb_postcode (
+    id integer,
+    postcode character varying(9),
+    geometry geometry,
+    CONSTRAINT enforce_dims_geometry CHECK ((st_ndims(geometry) = 2)),
+    CONSTRAINT enforce_srid_geometry CHECK ((st_srid(geometry) = 4326))
+);
+
+DROP TABLE IF EXISTS us_postcode;
+CREATE TABLE us_postcode (
+    postcode text,
+    x double precision,
+    y double precision
+);

lib-sql/tables.sql

@@ -35,8 +35,6 @@ GRANT UPDATE ON new_query_log TO "{www-user}" ;
 GRANT SELECT ON new_query_log TO "{www-user}" ;
 
 GRANT SELECT ON TABLE country_name TO "{www-user}";
-GRANT SELECT ON TABLE gb_postcode TO "{www-user}";
-GRANT SELECT ON TABLE us_postcode TO "{www-user}";
 
 drop table IF EXISTS word;
 CREATE TABLE word (

nominatim/cli.py

@@ -2,31 +2,19 @@
 Command-line interface to the Nominatim functions for import, update,
 database administration and querying.
 """
-import datetime as dt
-import os
-import socket
-import sys
-import time
-import argparse
 import logging
+import os
+import sys
+import argparse
 from pathlib import Path
 
 from .config import Configuration
-from .tools.exec_utils import run_legacy_script, run_api_script, run_php_server
+from .tools.exec_utils import run_legacy_script, run_php_server
-from .db.connection import connect
-from .db import status
 from .errors import UsageError
+from . import clicmd
 
 LOG = logging.getLogger()
 
-def _num_system_cpus():
-    try:
-        cpus = len(os.sched_getaffinity(0))
-    except NotImplementedError:
-        cpus = None
-
-    return cpus or os.cpu_count()
-
 
 class CommandlineParser:
     """ Wraps some of the common functions for parsing the command line
@@ -80,7 +68,8 @@ class CommandlineParser:
             self.parser.print_help()
             return 1
 
-        for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'data_dir', 'phpcgi_path'):
+        for arg in ('module_dir', 'osm2pgsql_path', 'phplib_dir', 'sqllib_dir',
+                    'data_dir', 'config_dir', 'phpcgi_path'):
             setattr(args, arg, Path(kwargs[arg]))
         args.project_dir = Path(args.project_dir).resolve()
 
@@ -89,7 +78,7 @@ class CommandlineParser:
                             datefmt='%Y-%m-%d %H:%M:%S',
                             level=max(4 - args.verbose, 1) * 10)
 
-        args.config = Configuration(args.project_dir, args.data_dir / 'settings')
+        args.config = Configuration(args.project_dir, args.config_dir)
 
         log = logging.getLogger()
         log.warning('Using project directory: %s', str(args.project_dir))
@@ -105,16 +94,6 @@ class CommandlineParser:
         return 1
 
 
-def _osm2pgsql_options_from_args(args, default_cache, default_threads):
-    """ Set up the stanadrd osm2pgsql from the command line arguments.
-    """
-    return dict(osm2pgsql=args.osm2pgsql_path,
-                osm2pgsql_cache=args.osm2pgsql_cache or default_cache,
-                osm2pgsql_style=args.config.get_import_style_file(),
-                threads=args.threads or default_threads,
-                dsn=args.config.get_libpq_dsn(),
-                flatnode_file=args.config.FLATNODE_FILE)
-
 ##### Subcommand classes
 #
 # Each class needs to implement two functions: add_args() adds the CLI parameters
@@ -237,153 +216,6 @@ class SetupSpecialPhrases:
         return run_legacy_script('specialphrases.php', '--wiki-import', nominatim_env=args)
 
 
-class UpdateReplication:
-    """\
-    Update the database using an online replication service.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        group = parser.add_argument_group('Arguments for initialisation')
-        group.add_argument('--init', action='store_true',
-                           help='Initialise the update process')
-        group.add_argument('--no-update-functions', dest='update_functions',
-                           action='store_false',
-                           help="""Do not update the trigger function to
-                                   support differential updates.""")
-        group = parser.add_argument_group('Arguments for updates')
-        group.add_argument('--check-for-updates', action='store_true',
-                           help='Check if new updates are available and exit')
-        group.add_argument('--once', action='store_true',
-                           help="""Download and apply updates only once. When
-                                   not set, updates are continuously applied""")
-        group.add_argument('--no-index', action='store_false', dest='do_index',
-                           help="""Do not index the new data. Only applicable
-                                   together with --once""")
-        group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
-                           help='Size of cache to be used by osm2pgsql (in MB)')
-        group = parser.add_argument_group('Download parameters')
-        group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,
-                           help='Set timeout for file downloads.')
-
-    @staticmethod
-    def _init_replication(args):
-        from .tools import replication, refresh
-
-        socket.setdefaulttimeout(args.socket_timeout)
-
-        LOG.warning("Initialising replication updates")
-        conn = connect(args.config.get_libpq_dsn())
-        replication.init_replication(conn, base_url=args.config.REPLICATION_URL)
-        if args.update_functions:
-            LOG.warning("Create functions")
-            refresh.create_functions(conn, args.config, args.data_dir,
-                                     True, False)
-        conn.close()
-        return 0
-
-
-    @staticmethod
-    def _check_for_updates(args):
-        from .tools import replication
-
-        conn = connect(args.config.get_libpq_dsn())
-        ret = replication.check_for_updates(conn, base_url=args.config.REPLICATION_URL)
-        conn.close()
-        return ret
-
-    @staticmethod
-    def _report_update(batchdate, start_import, start_index):
-        def round_time(delta):
-            return dt.timedelta(seconds=int(delta.total_seconds()))
-
-        end = dt.datetime.now(dt.timezone.utc)
-        LOG.warning("Update completed. Import: %s. %sTotal: %s. Remaining backlog: %s.",
-                    round_time((start_index or end) - start_import),
-                    "Indexing: {} ".format(round_time(end - start_index))
-                    if start_index else '',
-                    round_time(end - start_import),
-                    round_time(end - batchdate))
-
-    @staticmethod
-    def _update(args):
-        from .tools import replication
-        from .indexer.indexer import Indexer
-
-        params = _osm2pgsql_options_from_args(args, 2000, 1)
-        params.update(base_url=args.config.REPLICATION_URL,
-                      update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'),
-                      import_file=args.project_dir / 'osmosischange.osc',
-                      max_diff_size=args.config.get_int('REPLICATION_MAX_DIFF'),
-                      indexed_only=not args.once)
-
-        # Sanity check to not overwhelm the Geofabrik servers.
-        if 'download.geofabrik.de'in params['base_url']\
-           and params['update_interval'] < 86400:
-            LOG.fatal("Update interval too low for download.geofabrik.de.\n"
-                      "Please check install documentation "
-                      "(https://nominatim.org/release-docs/latest/admin/Import-and-Update#"
-                      "setting-up-the-update-process).")
-            raise UsageError("Invalid replication update interval setting.")
-
-        if not args.once:
-            if not args.do_index:
-                LOG.fatal("Indexing cannot be disabled when running updates continuously.")
-                raise UsageError("Bad argument '--no-index'.")
-            recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
-
-        while True:
-            conn = connect(args.config.get_libpq_dsn())
-            start = dt.datetime.now(dt.timezone.utc)
-            state = replication.update(conn, params)
-            if state is not replication.UpdateState.NO_CHANGES:
-                status.log_status(conn, start, 'import')
-            batchdate, _, _ = status.get_status(conn)
-            conn.close()
-
-            if state is not replication.UpdateState.NO_CHANGES and args.do_index:
-                index_start = dt.datetime.now(dt.timezone.utc)
-                indexer = Indexer(args.config.get_libpq_dsn(),
-                                  args.threads or 1)
-                indexer.index_boundaries(0, 30)
-                indexer.index_by_rank(0, 30)
-
-                conn = connect(args.config.get_libpq_dsn())
-                status.set_indexed(conn, True)
-                status.log_status(conn, index_start, 'index')
-                conn.close()
-            else:
-                index_start = None
-
-            if LOG.isEnabledFor(logging.WARNING):
-                UpdateReplication._report_update(batchdate, start, index_start)
-
-            if args.once:
-                break
-
-            if state is replication.UpdateState.NO_CHANGES:
-                LOG.warning("No new changes. Sleeping for %d sec.", recheck_interval)
-                time.sleep(recheck_interval)
-
-        return state.value
-
-    @staticmethod
-    def run(args):
-        try:
-            import osmium # pylint: disable=W0611
-        except ModuleNotFoundError:
-            LOG.fatal("pyosmium not installed. Replication functions not available.\n"
-                      "To install pyosmium via pip: pip3 install osmium")
-            return 1
-
-        if args.init:
-            return UpdateReplication._init_replication(args)
-
-        if args.check_for_updates:
-            return UpdateReplication._check_for_updates(args)
-
-        return UpdateReplication._update(args)
-
 class UpdateAddData:
     """\
     Add additional data from a file or an online source.
@@ -434,157 +266,6 @@ class UpdateAddData:
         return run_legacy_script(*params, nominatim_env=args)
 
 
-class UpdateIndex:
-    """\
-    Reindex all new and modified data.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        group = parser.add_argument_group('Filter arguments')
-        group.add_argument('--boundaries-only', action='store_true',
-                           help="""Index only administrative boundaries.""")
-        group.add_argument('--no-boundaries', action='store_true',
-                           help="""Index everything except administrative boundaries.""")
-        group.add_argument('--minrank', '-r', type=int, metavar='RANK', default=0,
-                           help='Minimum/starting rank')
-        group.add_argument('--maxrank', '-R', type=int, metavar='RANK', default=30,
-                           help='Maximum/finishing rank')
-
-    @staticmethod
-    def run(args):
-        from .indexer.indexer import Indexer
-
-        indexer = Indexer(args.config.get_libpq_dsn(),
-                          args.threads or _num_system_cpus() or 1)
-
-        if not args.no_boundaries:
-            indexer.index_boundaries(args.minrank, args.maxrank)
-        if not args.boundaries_only:
-            indexer.index_by_rank(args.minrank, args.maxrank)
-
-        if not args.no_boundaries and not args.boundaries_only \
-           and args.minrank == 0 and args.maxrank == 30:
-            conn = connect(args.config.get_libpq_dsn())
-            status.set_indexed(conn, True)
-            conn.close()
-
-        return 0
-
-
-class UpdateRefresh:
-    """\
-    Recompute auxiliary data used by the indexing process.
-
-    These functions must not be run in parallel with other update commands.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        group = parser.add_argument_group('Data arguments')
-        group.add_argument('--postcodes', action='store_true',
-                           help='Update postcode centroid table')
-        group.add_argument('--word-counts', action='store_true',
-                           help='Compute frequency of full-word search terms')
-        group.add_argument('--address-levels', action='store_true',
-                           help='Reimport address level configuration')
-        group.add_argument('--functions', action='store_true',
-                           help='Update the PL/pgSQL functions in the database')
-        group.add_argument('--wiki-data', action='store_true',
-                           help='Update Wikipedia/data importance numbers.')
-        group.add_argument('--importance', action='store_true',
-                           help='Recompute place importances (expensive!)')
-        group.add_argument('--website', action='store_true',
-                           help='Refresh the directory that serves the scripts for the web API')
-        group = parser.add_argument_group('Arguments for function refresh')
-        group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
-                           help='Do not enable code for propagating updates')
-        group.add_argument('--enable-debug-statements', action='store_true',
-                           help='Enable debug warning statements in functions')
-
-    @staticmethod
-    def run(args):
-        from .tools import refresh
-
-        if args.postcodes:
-            LOG.warning("Update postcodes centroid")
-            conn = connect(args.config.get_libpq_dsn())
-            refresh.update_postcodes(conn, args.data_dir)
-            conn.close()
-
-        if args.word_counts:
-            LOG.warning('Recompute frequency of full-word search terms')
-            conn = connect(args.config.get_libpq_dsn())
-            refresh.recompute_word_counts(conn, args.data_dir)
-            conn.close()
-
-        if args.address_levels:
-            cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
-            LOG.warning('Updating address levels from %s', cfg)
-            conn = connect(args.config.get_libpq_dsn())
-            refresh.load_address_levels_from_file(conn, cfg)
-            conn.close()
-
-        if args.functions:
-            LOG.warning('Create functions')
-            conn = connect(args.config.get_libpq_dsn())
-            refresh.create_functions(conn, args.config, args.data_dir,
-                                     args.diffs, args.enable_debug_statements)
-            conn.close()
-
-        if args.wiki_data:
-            run_legacy_script('setup.php', '--import-wikipedia-articles',
-                              nominatim_env=args, throw_on_fail=True)
-        # Attention: importance MUST come after wiki data import.
-        if args.importance:
-            run_legacy_script('update.php', '--recompute-importance',
-                              nominatim_env=args, throw_on_fail=True)
-        if args.website:
-            run_legacy_script('setup.php', '--setup-website',
-                              nominatim_env=args, throw_on_fail=True)
-
-        return 0
-
-
-class AdminCheckDatabase:
-    """\
-    Check that the database is complete and operational.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        pass # No options
-
-    @staticmethod
-    def run(args):
-        return run_legacy_script('check_import_finished.php', nominatim_env=args)
-
-
-class AdminWarm:
-    """\
-    Warm database caches for search and reverse queries.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        group = parser.add_argument_group('Target arguments')
-        group.add_argument('--search-only', action='store_const', dest='target',
-                           const='search',
-                           help="Only pre-warm tables for search queries")
-        group.add_argument('--reverse-only', action='store_const', dest='target',
-                           const='reverse',
-                           help="Only pre-warm tables for reverse queries")
-
-    @staticmethod
-    def run(args):
-        params = ['warm.php']
-        if args.target == 'reverse':
-            params.append('--reverse-only')
-        if args.target == 'search':
-            params.append('--search-only')
-        return run_legacy_script(*params, nominatim_env=args)
-
-
 class QueryExport:
     """\
     Export addresses as CSV file from the database.
@@ -662,246 +343,6 @@ class AdminServe:
     def run(args):
         run_php_server(args.server, args.project_dir / 'website')
 
-STRUCTURED_QUERY = (
-    ('street', 'housenumber and street'),
-    ('city', 'city, town or village'),
-    ('county', 'county'),
-    ('state', 'state'),
-    ('country', 'country'),
-    ('postalcode', 'postcode')
-)
-
-EXTRADATA_PARAMS = (
-    ('addressdetails', 'Include a breakdown of the address into elements.'),
-    ('extratags', """Include additional information if available
-                     (e.g. wikipedia link, opening hours)."""),
-    ('namedetails', 'Include a list of alternative names.')
-)
-
-DETAILS_SWITCHES = (
-    ('addressdetails', 'Include a breakdown of the address into elements.'),
-    ('keywords', 'Include a list of name keywords and address keywords.'),
-    ('linkedplaces', 'Include a details of places that are linked with this one.'),
-    ('hierarchy', 'Include details of places lower in the address hierarchy.'),
-    ('group_hierarchy', 'Group the places by type.'),
-    ('polygon_geojson', 'Include geometry of result.')
-)
-
-def _add_api_output_arguments(parser):
-    group = parser.add_argument_group('Output arguments')
-    group.add_argument('--format', default='jsonv2',
-                       choices=['xml', 'json', 'jsonv2', 'geojson', 'geocodejson'],
-                       help='Format of result')
-    for name, desc in EXTRADATA_PARAMS:
-        group.add_argument('--' + name, action='store_true', help=desc)
-
-    group.add_argument('--lang', '--accept-language', metavar='LANGS',
-                       help='Preferred language order for presenting search results')
-    group.add_argument('--polygon-output',
-                       choices=['geojson', 'kml', 'svg', 'text'],
-                       help='Output geometry of results as a GeoJSON, KML, SVG or WKT.')
-    group.add_argument('--polygon-threshold', type=float, metavar='TOLERANCE',
-                       help="""Simplify output geometry.
-                               Parameter is difference tolerance in degrees.""")
-
-
-class APISearch:
-    """\
-    Execute API search query.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        group = parser.add_argument_group('Query arguments')
-        group.add_argument('--query',
-                           help='Free-form query string')
-        for name, desc in STRUCTURED_QUERY:
-            group.add_argument('--' + name, help='Structured query: ' + desc)
-
-        _add_api_output_arguments(parser)
-
-        group = parser.add_argument_group('Result limitation')
-        group.add_argument('--countrycodes', metavar='CC,..',
-                           help='Limit search results to one or more countries.')
-        group.add_argument('--exclude_place_ids', metavar='ID,..',
-                           help='List of search object to be excluded')
-        group.add_argument('--limit', type=int,
-                           help='Limit the number of returned results')
-        group.add_argument('--viewbox', metavar='X1,Y1,X2,Y2',
-                           help='Preferred area to find search results')
-        group.add_argument('--bounded', action='store_true',
-                           help='Strictly restrict results to viewbox area')
-
-        group = parser.add_argument_group('Other arguments')
-        group.add_argument('--no-dedupe', action='store_false', dest='dedupe',
-                           help='Do not remove duplicates from the result list')
-
-
-    @staticmethod
-    def run(args):
-        if args.query:
-            params = dict(q=args.query)
-        else:
-            params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
-
-        for param, _ in EXTRADATA_PARAMS:
-            if getattr(args, param):
-                params[param] = '1'
-        for param in ('format', 'countrycodes', 'exclude_place_ids', 'limit', 'viewbox'):
-            if getattr(args, param):
-                params[param] = getattr(args, param)
-        if args.lang:
-            params['accept-language'] = args.lang
-        if args.polygon_output:
-            params['polygon_' + args.polygon_output] = '1'
-        if args.polygon_threshold:
-            params['polygon_threshold'] = args.polygon_threshold
-        if args.bounded:
-            params['bounded'] = '1'
-        if not args.dedupe:
-            params['dedupe'] = '0'
-
-        return run_api_script('search', args.project_dir,
-                              phpcgi_bin=args.phpcgi_path, params=params)
-
-class APIReverse:
-    """\
-    Execute API reverse query.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        group = parser.add_argument_group('Query arguments')
-        group.add_argument('--lat', type=float, required=True,
-                           help='Latitude of coordinate to look up (in WGS84)')
-        group.add_argument('--lon', type=float, required=True,
-                           help='Longitude of coordinate to look up (in WGS84)')
-        group.add_argument('--zoom', type=int,
-                           help='Level of detail required for the address')
-
-        _add_api_output_arguments(parser)
-
-
-    @staticmethod
-    def run(args):
-        params = dict(lat=args.lat, lon=args.lon)
-        if args.zoom is not None:
-            params['zoom'] = args.zoom
-
-        for param, _ in EXTRADATA_PARAMS:
-            if getattr(args, param):
-                params[param] = '1'
-        if args.format:
-            params['format'] = args.format
-        if args.lang:
-            params['accept-language'] = args.lang
-        if args.polygon_output:
-            params['polygon_' + args.polygon_output] = '1'
-        if args.polygon_threshold:
-            params['polygon_threshold'] = args.polygon_threshold
-
-        return run_api_script('reverse', args.project_dir,
-                              phpcgi_bin=args.phpcgi_path, params=params)
-
-
-class APILookup:
-    """\
-    Execute API reverse query.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        group = parser.add_argument_group('Query arguments')
-        group.add_argument('--id', metavar='OSMID',
-                           action='append', required=True, dest='ids',
-                           help='OSM id to lookup in format <NRW><id> (may be repeated)')
-
-        _add_api_output_arguments(parser)
-
-
-    @staticmethod
-    def run(args):
-        params = dict(osm_ids=','.join(args.ids))
-
-        for param, _ in EXTRADATA_PARAMS:
-            if getattr(args, param):
-                params[param] = '1'
-        if args.format:
-            params['format'] = args.format
-        if args.lang:
-            params['accept-language'] = args.lang
-        if args.polygon_output:
-            params['polygon_' + args.polygon_output] = '1'
-        if args.polygon_threshold:
-            params['polygon_threshold'] = args.polygon_threshold
-
-        return run_api_script('lookup', args.project_dir,
-                              phpcgi_bin=args.phpcgi_path, params=params)
-
-
-class APIDetails:
-    """\
-    Execute API lookup query.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        group = parser.add_argument_group('Query arguments')
-        objs = group.add_mutually_exclusive_group(required=True)
-        objs.add_argument('--node', '-n', type=int,
-                          help="Look up the OSM node with the given ID.")
-        objs.add_argument('--way', '-w', type=int,
-                          help="Look up the OSM way with the given ID.")
-        objs.add_argument('--relation', '-r', type=int,
-                          help="Look up the OSM relation with the given ID.")
-        objs.add_argument('--place_id', '-p', type=int,
-                          help='Database internal identifier of the OSM object to look up.')
-        group.add_argument('--class', dest='object_class',
-                           help="""Class type to disambiguated multiple entries
-                                   of the same object.""")
-
-        group = parser.add_argument_group('Output arguments')
-        for name, desc in DETAILS_SWITCHES:
-            group.add_argument('--' + name, action='store_true', help=desc)
-        group.add_argument('--lang', '--accept-language', metavar='LANGS',
-                           help='Preferred language order for presenting search results')
-
-    @staticmethod
-    def run(args):
-        if args.node:
-            params = dict(osmtype='N', osmid=args.node)
-        elif args.way:
-            params = dict(osmtype='W', osmid=args.node)
-        elif args.relation:
-            params = dict(osmtype='R', osmid=args.node)
-        else:
-            params = dict(place_id=args.place_id)
-        if args.object_class:
-            params['class'] = args.object_class
-        for name, _ in DETAILS_SWITCHES:
-            params[name] = '1' if getattr(args, name) else '0'
-
-        return run_api_script('details', args.project_dir,
-                              phpcgi_bin=args.phpcgi_path, params=params)
-
-
-class APIStatus:
-    """\
-    Execute API status query.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        group = parser.add_argument_group('API parameters')
-        group.add_argument('--format', default='text', choices=['text', 'json'],
-                           help='Format of result')
-
-    @staticmethod
-    def run(args):
-        return run_api_script('status', args.project_dir,
-                              phpcgi_bin=args.phpcgi_path,
-                              params=dict(format=args.format))
-
 
 def nominatim(**kwargs):
     """\
@@ -912,26 +353,25 @@ def nominatim(**kwargs):
 
     parser.add_subcommand('import', SetupAll)
     parser.add_subcommand('freeze', SetupFreeze)
-    parser.add_subcommand('replication', UpdateReplication)
+    parser.add_subcommand('replication', clicmd.UpdateReplication)
-
-    parser.add_subcommand('check-database', AdminCheckDatabase)
-    parser.add_subcommand('warm', AdminWarm)
 
     parser.add_subcommand('special-phrases', SetupSpecialPhrases)
 
     parser.add_subcommand('add-data', UpdateAddData)
-    parser.add_subcommand('index', UpdateIndex)
+    parser.add_subcommand('index', clicmd.UpdateIndex)
-    parser.add_subcommand('refresh', UpdateRefresh)
+    parser.add_subcommand('refresh', clicmd.UpdateRefresh)
+
+    parser.add_subcommand('admin', clicmd.AdminFuncs)
 
     parser.add_subcommand('export', QueryExport)
     parser.add_subcommand('serve', AdminServe)
 
     if kwargs.get('phpcgi_path'):
-        parser.add_subcommand('search', APISearch)
+        parser.add_subcommand('search', clicmd.APISearch)
-        parser.add_subcommand('reverse', APIReverse)
+        parser.add_subcommand('reverse', clicmd.APIReverse)
-        parser.add_subcommand('lookup', APILookup)
+        parser.add_subcommand('lookup', clicmd.APILookup)
-        parser.add_subcommand('details', APIDetails)
+        parser.add_subcommand('details', clicmd.APIDetails)
-        parser.add_subcommand('status', APIStatus)
+        parser.add_subcommand('status', clicmd.APIStatus)
     else:
         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 

nominatim/clicmd/__init__.py

@@ -0,0 +1,9 @@
+"""
+Subcommand definitions for the command-line tool.
+"""
+
+from .replication import UpdateReplication
+from .api import APISearch, APIReverse, APILookup, APIDetails, APIStatus
+from .index import UpdateIndex
+from .refresh import UpdateRefresh
+from .admin import AdminFuncs

nominatim/clicmd/admin.py

@@ -0,0 +1,64 @@
+"""
+Implementation of the 'admin' subcommand.
+"""
+from ..tools.exec_utils import run_legacy_script
+from ..db.connection import connect
+
+# Do not repeat documentation of subcommand classes.
+# pylint: disable=C0111
+# Using non-top-level imports to avoid eventually unused imports.
+# pylint: disable=E0012,C0415
+
+class AdminFuncs:
+    """\
+    Analyse and maintain the database.
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('Admin task arguments')
+        group.add_argument('--warm', action='store_true',
+                           help='Warm database caches for search and reverse queries.')
+        group.add_argument('--check-database', action='store_true',
+                           help='Check that the database is complete and operational.')
+        group.add_argument('--analyse-indexing', action='store_true',
+                           help='Print performance analysis of the indexing process.')
+        group = parser.add_argument_group('Arguments for cache warming')
+        group.add_argument('--search-only', action='store_const', dest='target',
+                           const='search',
+                           help="Only pre-warm tables for search queries")
+        group.add_argument('--reverse-only', action='store_const', dest='target',
+                           const='reverse',
+                           help="Only pre-warm tables for reverse queries")
+        group = parser.add_argument_group('Arguments for index anaysis')
+        mgroup = group.add_mutually_exclusive_group()
+        mgroup.add_argument('--osm-id', type=str,
+                            help='Analyse indexing of the given OSM object')
+        mgroup.add_argument('--place-id', type=int,
+                            help='Analyse indexing of the given Nominatim object')
+
+    @staticmethod
+    def run(args):
+        from ..tools import admin
+        if args.warm:
+            AdminFuncs._warm(args)
+
+        if args.check_database:
+            run_legacy_script('check_import_finished.php', nominatim_env=args)
+
+        if args.analyse_indexing:
+            conn = connect(args.config.get_libpq_dsn())
+            admin.analyse_indexing(conn, osm_id=args.osm_id, place_id=args.place_id)
+            conn.close()
+
+        return 0
+
+
+    @staticmethod
+    def _warm(args):
+        params = ['warm.php']
+        if args.target == 'reverse':
+            params.append('--reverse-only')
+        if args.target == 'search':
+            params.append('--search-only')
+        return run_legacy_script(*params, nominatim_env=args)