Replace lettuce with behave in tests. Use python3 Split import and (python,php) tests into two travis executions

.travis.yml

@@ -1,24 +1,30 @@
 ---
-language:
-  - 'python'
 sudo: required
 dist: trusty
-before_install:
-  - git submodule update --init --recursive
+language: python
+python:
+  - "3.6"
+addons:
+  postgresql: "9.6"
+git:
+  depth: 3
+env:
+  - TEST_SUITE=tests
+  - TEST_SUITE=monaco
 install:
   - vagrant/install-on-travis-ci.sh
 before_script:
-  - cd $TRAVIS_BUILD_DIR/build
-  - wget --no-verbose --output-document=../data/monaco.osm.pbf http://download.geofabrik.de/europe/monaco-latest.osm.pbf
-  - ./utils/setup.php --osm-file ../data/monaco.osm.pbf --osm2pgsql-cache 1000 --all 2>&1 | grep -v 'ETA (seconds)'
-  - ./utils/specialphrases.php --countries > ../data/specialphrases_countries.sql
-  - psql -d nominatim -f ../data/specialphrases_countries.sql
+  - psql -U postgres -c "create extension postgis"
 script:
-  - cd $TRAVIS_BUILD_DIR/tests-php
-  - phpunit ./
-  - cd $TRAVIS_BUILD_DIR/tests
-  - #lettuce features/api --verbosity=1
-  - lettuce features/db --verbosity=1 -t -Fail -t -Tiger -t -poldi-only
-  - lettuce features/osm2pgsql --verbosity=1 -t -Fail
+  - cd $TRAVIS_BUILD_DIR/build
+  - if [[ $TEST_SUITE == "monaco" ]]; then wget --no-verbose --output-document=../data/monaco.osm.pbf http://download.geofabrik.de/europe/monaco-latest.osm.pbf; fi
+  - if [[ $TEST_SUITE == "monaco" ]]; then ./utils/setup.php --osm-file ../data/monaco.osm.pbf --osm2pgsql-cache 1000 --all 2>&1 | grep -v 'ETA (seconds)'; fi
+  - cd $TRAVIS_BUILD_DIR/test/php
+  - if [[ $TEST_SUITE == "tests" ]]; then phpunit ./ ; fi
+  - if [[ $TEST_SUITE == "tests" ]]; then phpcs --report-width=120 */**.php ; fi
+  - cd $TRAVIS_BUILD_DIR/test/bdd
+  - # behave --format=progress3 api
+  - if [[ $TEST_SUITE == "tests" ]]; then behave --format=progress3 db ; fi
+  - if [[ $TEST_SUITE == "tests" ]]; then behave --format=progress3 osm2pgsql ; fi
 notifications:
   email: false

VAGRANT.md

@@ -49,8 +49,6 @@ is.
   cd build
     wget --no-verbose --output-document=../data/monaco.osm.pbf http://download.geofabrik.de/europe/monaco-latest.osm.pbf
     ./utils/setup.php --osm-file ../data/monaco.osm.pbf --osm2pgsql-cache 1000 --all 2>&1 | tee monaco.$$.log
-    ./utils/specialphrases.php --countries > ../data/specialphrases_countries.sql
-    psql -d nominatim -f ../data/specialphrases_countries.sql
     ```
 
   To repeat an import you'd need to delete the database first

test/README.md

@@ -11,7 +11,7 @@ Prerequisites
 
 To get the prerequisites on a a fresh Ubuntu LTS 16.04 run:
 
-     [sudo] apt-get install python3-dev python3-pip python3-psycopg2 python3-tidylib phpunit
+     [sudo] apt-get install python3-dev python3-pip python3-psycopg2 python3-tidylib phpunit php-cgi
      pip3 install --user behave nose
 
 

vagrant/install-on-travis-ci.sh

@@ -1,35 +1,28 @@
 #!/bin/bash
 
-# This script runs in a travis-ci.org (or .com) virtual machine
+# This script runs in a travis-ci.org virtual machine
 # https://docs.travis-ci.com/user/trusty-ci-environment/
 # Ubuntu 14 (trusty)
 # user 'travis'
-# $TRAVIS_BUILD_DIR is /home/travis/build/twain47/Nominatim/, for more see
+# $TRAVIS_BUILD_DIR is /home/travis/build/twain47/Nominatim/, for others see
 #   https://docs.travis-ci.com/user/environment-variables/#Default-Environment-Variables
-# Postgres 9.2 installed and started. role 'travis' already superuser
-# Python 2.7.10, pip 7.1.2
+# Postgres 9.6 installed and started. role 'travis' already superuser
+# Python 3.6
+# Travis has a 4 MB, 10000 line output limit, so where possible we run script --quiet
 
-# Travis has a 4 MB, 10000 line output limit, so where possible we supress
-#  output from installation scripts
-# Travis strips color from the output
-
-sudo service postgresql stop
 
 sudo apt-get update -qq
 sudo apt-get install -y -qq libboost-dev libboost-system-dev \
                             libboost-filesystem-dev libexpat1-dev zlib1g-dev libxml2-dev\
                             libbz2-dev libpq-dev libgeos-c1 libgeos++-dev libproj-dev \
-                        postgresql-server-dev-9.3 postgresql-9.3-postgis-2.1 postgresql-contrib-9.3 \
+                            postgresql-server-dev-9.6 postgresql-9.6-postgis-2.3 postgresql-contrib-9.6 \
                             apache2 php5 php5-pgsql php-pear php-db
 
-sudo apt-get install -y -qq python-Levenshtein python-shapely \
-                        python-psycopg2 tidy python-nose python-tidylib \
-                        python-numpy phpunit
+sudo apt-get install -y -qq python3-dev python3-pip python3-psycopg2 phpunit php5-cgi
 
-sudo -H pip install --quiet 'setuptools>=23.0.0' lettuce==0.2.18 'six>=1.9' haversine
-sudo pear install PHP_CodeSniffer
+pip3 install --quiet behave nose pytidylib psycopg2
+sudo pear -q install PHP_CodeSniffer
 
-sudo service postgresql restart
 sudo -u postgres createuser -S www-data
 
 # Make sure that system servers can read from the home directory:
@@ -62,4 +55,6 @@ make
 tee settings/local.php << EOF
 <?php
  @define('CONST_Website_BaseURL', '/nominatim/');
+ @define('CONST_Database_DSN', 'pgsql://@/test_api_nominatim');
+ @define('CONST_Wikipedia_Data_Path', CONST_BasePath.'/test/testdb');
 EOF

vagrant/install-on-ubuntu-16.sh

@@ -18,8 +18,8 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 # Make sure all packages are are up-to-date by running:
 #
 
+    sudo apt-get -o DPkg::options::="--force-confdef" -o DPkg::options::="--force-confold" --force-yes -fuy install grub-pc #DOCS:
     sudo apt-get update -qq
-    sudo apt-get -o Dpkg::Options::="--force-confold" --force-yes -fuy dist-upgrade #DOCS:    sudo apt-get upgrade
 
 # Now you can install all packages needed for Nominatim:
 
@@ -33,11 +33,9 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 # If you want to run the test suite, you need to install the following
 # additional packages:
 
-    sudo apt-get install -y python-dev python-pip python-levenshtein python-shapely \
-                            python-psycopg2 tidy python-nose python-tidylib \
-                            python-numpy phpunit
+    sudo apt-get install -y python3-dev python3-pip python3-psycopg2 python3-tidylib phpunit
 
-    pip install --user lettuce==0.2.18 six==1.7 haversine
+    pip3 install --user behave nose # urllib3
     sudo pear install PHP_CodeSniffer
 
 #