From 2e5bdb87941752aa12b049bb97f8508ac125c61d Mon Sep 17 00:00:00 2001 From: marc tobias Date: Sat, 20 Jun 2020 03:48:07 +0200 Subject: [PATCH] Put install instructions of test tools into separate docs/ markdown file --- docs/admin/Advanced-Installations.md | 2 +- .../admin/{Import-and-Update.md => Import.md} | 69 +------------------ docs/admin/Installation.md | 2 +- docs/admin/Update.md | 67 ++++++++++++++++++ docs/develop/Documentation.md | 3 + docs/develop/Setup.md | 46 +++++++++++++ docs/mkdocs.yml | 4 +- vagrant/Install-on-Centos-7.sh | 14 +--- vagrant/Install-on-Centos-8.sh | 14 +--- vagrant/Install-on-Ubuntu-18.sh | 14 +--- vagrant/Install-on-Ubuntu-20.sh | 8 +-- 11 files changed, 126 insertions(+), 117 deletions(-) rename docs/admin/{Import-and-Update.md => Import.md} (80%) create mode 100644 docs/admin/Update.md create mode 100644 docs/develop/Setup.md diff --git a/docs/admin/Advanced-Installations.md b/docs/admin/Advanced-Installations.md index 8486b6a2..ee3a7992 100644 --- a/docs/admin/Advanced-Installations.md +++ b/docs/admin/Advanced-Installations.md @@ -105,5 +105,5 @@ This will get diffs from the replication server, import diffs and index the data ## Verification and further setup -Instructions for import verification and other details like importing Wikidata can be found in [import and update page](Import-and-Update.md) +Instructions for import verification and other details like importing Wikidata can be found in [import page](Import.md) diff --git a/docs/admin/Import-and-Update.md b/docs/admin/Import.md similarity index 80% rename from docs/admin/Import-and-Update.md rename to docs/admin/Import.md index 47516a2b..bcc33af8 100644 --- a/docs/admin/Import-and-Update.md +++ b/docs/admin/Import.md @@ -1,4 +1,4 @@ -# Importing and Updating the Database +# Importing the Database The following instructions explain how to create a Nominatim database from an OSM planet file and how to keep the database up to date. It @@ -253,70 +253,3 @@ entire US adds about 10GB to your database. ``` -## Updates - -There are many different ways to update your Nominatim database. -The following section describes how to keep it up-to-date with Pyosmium. -For a list of other methods see the output of `./utils/update.php --help`. - -!!! warning - If you have configured a flatnode file for the import, then you - need to keep this flatnode file around for updates as well. - -#### Installing the newest version of Pyosmium - -It is recommended to install Pyosmium via pip. Make sure to use python3. -Run (as the same user who will later run the updates): - -```sh -pip3 install --user osmium -``` - -Nominatim needs a tool called `pyosmium-get-updates` which comes with -Pyosmium. You need to tell Nominatim where to find it. Add the -following line to your `settings/local.php`: - - @define('CONST_Pyosmium_Binary', '/home/user/.local/bin/pyosmium-get-changes'); - -The path above is fine if you used the `--user` parameter with pip. -Replace `user` with your user name. - -#### Setting up the update process - -Next the update needs to be initialised. By default Nominatim is configured -to update using the global minutely diffs. - -If you want a different update source you will need to add some settings -to `settings/local.php`. For example, to use the daily country extracts -diffs for Ireland from Geofabrik add the following: - - // base URL of the replication service - @define('CONST_Replication_Url', 'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates'); - // How often upstream publishes diffs - @define('CONST_Replication_Update_Interval', '86400'); - // How long to sleep if no update found yet - @define('CONST_Replication_Recheck_Interval', '900'); - -To set up the update process now run the following command: - - ./utils/update.php --init-updates - -It outputs the date where updates will start. Recheck that this date is -what you expect. - -The `--init-updates` command needs to be rerun whenever the replication service -is changed. - -#### Updating Nominatim - -The following command will keep your database constantly up to date: - - ./utils/update.php --import-osmosis-all - -(Note that even though the old name "import-osmosis-all" has been kept for -compatibility reasons, Osmosis is not required to run this - it uses pyosmium -behind the scenes.) - -If you have imported multiple country extracts and want to keep them -up-to-date, [Advanced installations section](Advanced-Installations.md) contains instructions -to set up and update multiple country extracts. diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index cf994c4c..bc97212e 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -188,4 +188,4 @@ Restart the nginx and php5-fpm services and the website should now be available at `http://localhost/`. -Now continue with [importing the database](Import-and-Update.md). +Now continue with [importing the database](Import.md). diff --git a/docs/admin/Update.md b/docs/admin/Update.md new file mode 100644 index 00000000..8cc2b343 --- /dev/null +++ b/docs/admin/Update.md @@ -0,0 +1,67 @@ +# Updating the Database + +There are many different ways to update your Nominatim database. +The following section describes how to keep it up-to-date with Pyosmium. +For a list of other methods see the output of `./utils/update.php --help`. + +!!! warning + If you have configured a flatnode file for the import, then you + need to keep this flatnode file around for updates as well. + +#### Installing the newest version of Pyosmium + +It is recommended to install Pyosmium via pip. Make sure to use python3. +Run (as the same user who will later run the updates): + +```sh +pip3 install --user osmium +``` + +Nominatim needs a tool called `pyosmium-get-updates` which comes with +Pyosmium. You need to tell Nominatim where to find it. Add the +following line to your `settings/local.php`: + + @define('CONST_Pyosmium_Binary', '/home/user/.local/bin/pyosmium-get-changes'); + +The path above is fine if you used the `--user` parameter with pip. +Replace `user` with your user name. + +#### Setting up the update process + +Next the update needs to be initialised. By default Nominatim is configured +to update using the global minutely diffs. + +If you want a different update source you will need to add some settings +to `settings/local.php`. For example, to use the daily country extracts +diffs for Ireland from Geofabrik add the following: + + // base URL of the replication service + @define('CONST_Replication_Url', 'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates'); + // How often upstream publishes diffs + @define('CONST_Replication_Update_Interval', '86400'); + // How long to sleep if no update found yet + @define('CONST_Replication_Recheck_Interval', '900'); + +To set up the update process now run the following command: + + ./utils/update.php --init-updates + +It outputs the date where updates will start. Recheck that this date is +what you expect. + +The `--init-updates` command needs to be rerun whenever the replication service +is changed. + +#### Updating Nominatim + +The following command will keep your database constantly up to date: + + ./utils/update.php --import-osmosis-all + +(Note that even though the old name "import-osmosis-all" has been kept for +compatibility reasons, Osmosis is not required to run this - it uses pyosmium +behind the scenes.) + +If you have imported multiple country extracts and want to keep them +up-to-date, [Advanced installations section](Advanced-Installations.md) contains instructions +to set up and update multiple country extracts. diff --git a/docs/develop/Documentation.md b/docs/develop/Documentation.md index df8d2b1a..debe6858 100644 --- a/docs/develop/Documentation.md +++ b/docs/develop/Documentation.md @@ -8,6 +8,9 @@ To preview local changes, first install MkDocs pip3 install --user mkdocs ``` +If `mkdocs` can't be found after the installation, the $PATH might have not +be set correctly yet. Try opening a new terminal session. + Then go to the build directory and run diff --git a/docs/develop/Setup.md b/docs/develop/Setup.md new file mode 100644 index 00000000..53a084d2 --- /dev/null +++ b/docs/develop/Setup.md @@ -0,0 +1,46 @@ +# Setup Test Environment + +To test changes and contribute to Nominatim you should be able to run +the test suite(s). For many usecases it's enough to create a Vagrant +virtual machine (see `VAGRANT.md`), import one small country into the +database. + +## Prerequisites + +Nominatim supports a range of PHP versions and PHPUnit versions also +move fast. We try to test against the newest stable PHP release and +PHPUnit version even though we expect many Nominatim users will install +older version on their production servers. + +#### Ubuntu 20 + + sudo apt-get install -y phpunit php-codesniffer php-cgi + pip3 install --user behave nose + +#### Ubuntu 18 + + pip3 install --user behave nose + + sudo apt-get install -y composer php-cgi php-cli php-mbstring php-xml zip unzip + + composer global require "squizlabs/php_codesniffer=*" + sudo ln -s ~/.config/composer/vendor/bin/phpcs /usr/bin/ + + composer global require "phpunit/phpunit=8.*" + sudo ln -s ~/.config/composer/vendor/bin/phpunit /usr/bin/ + + +#### CentOS 7 or 8 + + sudo dnf install -y php-dom php-mbstring + pip3 install --user behave nose + + composer global require "squizlabs/php_codesniffer=*" + sudo ln -s ~/.config/composer/vendor/bin/phpcs /usr/bin/ + + composer global require "phpunit/phpunit=^7" + sudo ln -s ~/.config/composer/vendor/bin/phpunit /usr/bin/ + +## Run tests, code linter, code coverage + +See `README.md` in `test` subdirectory. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 16a67538..0a411306 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -16,7 +16,8 @@ pages: - 'FAQ': 'api/Faq.md' - 'Administration Guide': - 'Basic Installation': 'admin/Installation.md' - - 'Importing and Updating' : 'admin/Import-and-Update.md' + - 'Importing' : 'admin/Import.md' + - 'Updating' : 'admin/Update.md' - 'Nominatim UI' : 'admin/Setup-Nominatim-UI.md' - 'Advanced Installations' : 'admin/Advanced-Installations.md' - 'Migration from older Versions' : 'admin/Migration.md' @@ -26,6 +27,7 @@ pages: - 'OSM Data Import' : 'develop/Import.md' - 'Place Ranking' : 'develop/Ranking.md' - 'Postcodes' : 'develop/Postcodes.md' + - 'Setup Test Environment' : 'develop/Setup.md' - 'Documentation' : 'develop/Documentation.md' - 'External Data Sources': - 'Overview' : 'data-sources/overview.md' diff --git a/vagrant/Install-on-Centos-7.sh b/vagrant/Install-on-Centos-7.sh index e2592f65..7ce09727 100755 --- a/vagrant/Install-on-Centos-7.sh +++ b/vagrant/Install-on-Centos-7.sh @@ -48,18 +48,6 @@ pip3 install --user psycopg2 pytidylib -# If you want to run the test suite, you need to install the following -# additional packages: - -#DOCS: :::sh - sudo yum install -y php-dom php-mbstring - pip3 install --user behave nose - - composer global require "squizlabs/php_codesniffer=*" - sudo ln -s ~/.config/composer/vendor/bin/phpcs /usr/bin/ - - composer global require "phpunit/phpunit=7.*" - sudo ln -s ~/.config/composer/vendor/bin/phpunit /usr/bin/ # # System Configuration @@ -214,4 +202,4 @@ EOF # Nominatim is now ready to use. Continue with -# [importing a database from OSM data](../admin/Import-and-Update.md). +# [importing a database from OSM data](../admin/Import.md). diff --git a/vagrant/Install-on-Centos-8.sh b/vagrant/Install-on-Centos-8.sh index cf7371ed..75b7a558 100755 --- a/vagrant/Install-on-Centos-8.sh +++ b/vagrant/Install-on-Centos-8.sh @@ -41,18 +41,6 @@ pip3 install --user psycopg2 pytidylib -# If you want to run the test suite, you need to install the following -# additional packages: - -#DOCS: :::sh - sudo dnf install -y php-dom php-mbstring - pip3 install --user behave nose - - composer global require "squizlabs/php_codesniffer=*" - sudo ln -s ~/.config/composer/vendor/bin/phpcs /usr/bin/ - - composer global require "phpunit/phpunit=^7" - sudo ln -s ~/.config/composer/vendor/bin/phpunit /usr/bin/ # # System Configuration @@ -208,4 +196,4 @@ EOF # Nominatim is now ready to use. Continue with -# [importing a database from OSM data](../admin/Import-and-Update.md). +# [importing a database from OSM data](../admin/Import.md). diff --git a/vagrant/Install-on-Ubuntu-18.sh b/vagrant/Install-on-Ubuntu-18.sh index 02a1c738..34e6aaa0 100755 --- a/vagrant/Install-on-Ubuntu-18.sh +++ b/vagrant/Install-on-Ubuntu-18.sh @@ -33,18 +33,6 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: php-intl python3-setuptools python3-dev python3-pip \ python3-psycopg2 python3-tidylib git -# If you want to run the test suite, you need to install the following -# additional packages including the PHP package manager composer: - - pip3 install --user behave nose - - sudo apt-get install -y composer php-cgi php-cli php-mbstring php-xml zip unzip - - composer global require "squizlabs/php_codesniffer=*" - sudo ln -s ~/.config/composer/vendor/bin/phpcs /usr/bin/ - - composer global require "phpunit/phpunit=8.*" - sudo ln -s ~/.config/composer/vendor/bin/phpunit /usr/bin/ # # System Configuration @@ -173,4 +161,4 @@ EOF # Nominatim is now ready to use. Continue with -# [importing a database from OSM data](../admin/Import-and-Update.md). +# [importing a database from OSM data](../admin/Import.md). diff --git a/vagrant/Install-on-Ubuntu-20.sh b/vagrant/Install-on-Ubuntu-20.sh index c124b159..a7029be2 100644 --- a/vagrant/Install-on-Ubuntu-20.sh +++ b/vagrant/Install-on-Ubuntu-20.sh @@ -35,12 +35,6 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: php-intl python3-setuptools python3-dev python3-pip \ python3-psycopg2 python3-tidylib git -# If you want to run the test suite, you need to install the following -# additional packages: - - sudo apt-get install -y phpunit php-codesniffer php-cgi - - pip3 install --user behave nose # # System Configuration # ==================== @@ -168,4 +162,4 @@ EOF # Nominatim is now ready to use. Continue with -# [importing a database from OSM data](../admin/Import-and-Update.md). +# [importing a database from OSM data](../admin/Import.md).