osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-11-22 12:06:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2881 from lonvia/more-update-tests-for-osm2pgsql

Browse Source 
Experimental support for osm2pgsql flex output
This commit is contained in:
Sarah Hoffmann 2022-11-15 09:39:46 +01:00 committed by GitHub
commit f3f542e864
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 1784 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/actions/build-nominatim/action.yml vendored
View File

@ -9,6 +9,10 @@ inputs:
description: 'Additional options to hand to cmake'
required: false
default: ''
lua:
description: 'Version of Lua to use'
required: false
default: '5.3'
runs:
using: "composite"
@ -21,7 +25,7 @@ runs:
shell: bash
- name: Install prerequisites
run: |
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION}
if [ "x$UBUNTUVER" == "x18" ]; then
pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 datrie
else
@ -31,6 +35,7 @@ runs:
env:
UBUNTUVER: ${{ inputs.ubuntu }}
CMAKE_ARGS: ${{ inputs.cmake-args }}
LUA_VERSION: ${{ inputs.lua }}
- name: Configure
run: mkdir build && cd build && cmake $CMAKE_ARGS ../Nominatim

1
CMakeLists.txt
View File

@ -63,7 +63,6 @@ if (BUILD_IMPORTER AND BUILD_OSM2PGSQL)
endif()
set(BUILD_TESTS_SAVED "${BUILD_TESTS}")
set(BUILD_TESTS off)
set(WITH_LUA off CACHE BOOL "")
add_subdirectory(osm2pgsql)
set(BUILD_TESTS ${BUILD_TESTS_SAVED})
endif()

116
lib-sql/functions/place_triggers.sql
View File

@ -34,6 +34,11 @@ BEGIN
RETURN null;
END IF;
-- Remove the place from the list of places to be deleted
DELETE FROM place_to_be_deleted pdel
WHERE pdel.osm_type = NEW.osm_type and pdel.osm_id = NEW.osm_id
and pdel.class = NEW.class;
-- Have we already done this place?
SELECT * INTO existing
FROM place
@ -42,8 +47,6 @@ BEGIN
{% if debug %}RAISE WARNING 'Existing: %',existing.osm_id;{% endif %}
-- Handle a place changing type by removing the old data.
-- (This trigger is executed BEFORE INSERT of the NEW tuple.)
IF existing.osm_type IS NULL THEN
DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class;
END IF;
@ -187,15 +190,11 @@ BEGIN
END IF;
{% endif %}
IF existing.osm_type IS NOT NULL THEN
-- Pathological case caused by the triggerless copy into place during initial import
-- force delete even for large areas, it will be reinserted later
UPDATE place SET geometry = ST_SetSRID(ST_Point(0,0), 4326)
WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
and class = NEW.class and type = NEW.type;
DELETE FROM place
WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
and class = NEW.class and type = NEW.type;
IF existingplacex.osm_type is not NULL THEN
-- Mark any existing place for delete in the placex table
UPDATE placex SET indexed_status = 100
WHERE placex.osm_type = NEW.osm_type and placex.osm_id = NEW.osm_id
and placex.class = NEW.class and placex.type = NEW.type;
END IF;
-- Process it as a new insertion
@ -206,6 +205,27 @@ BEGIN
{% if debug %}RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name;{% endif %}
IF existing.osm_type is not NULL THEN
-- If there is already an entry in place, just update that, if necessary.
IF coalesce(existing.name, ''::hstore) != coalesce(NEW.name, ''::hstore)
or coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore)
or coalesce(existing.extratags, ''::hstore) != coalesce(NEW.extratags, ''::hstore)
or coalesce(existing.admin_level, 15) != coalesce(NEW.admin_level, 15)
or existing.geometry::text != NEW.geometry::text
THEN
UPDATE place
SET name = NEW.name,
address = NEW.address,
extratags = NEW.extratags,
admin_level = NEW.admin_level,
geometry = NEW.geometry
WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
and class = NEW.class and type = NEW.type;
END IF;
RETURN NULL;
END IF;
RETURN NEW;
END IF;
@ -321,35 +341,67 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION place_delete()
RETURNS TRIGGER
AS $$
DECLARE
has_rank BOOLEAN;
deferred BOOLEAN;
BEGIN
{% if debug %}RAISE WARNING 'Delete for % % %/%', OLD.osm_type, OLD.osm_id, OLD.class, OLD.type;{% endif %}
{% if debug %}RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type;{% endif %}
-- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
IF st_area(OLD.geometry) > 2 and st_isvalid(OLD.geometry) THEN
SELECT bool_or(not (rank_address = 0 or rank_address > 25)) as ranked FROM placex WHERE osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type INTO has_rank;
IF has_rank THEN
insert into import_polygon_delete (osm_type, osm_id, class, type) values (OLD.osm_type,OLD.osm_id,OLD.class,OLD.type);
RETURN NULL;
END IF;
deferred := ST_IsValid(OLD.geometry) and ST_Area(OLD.geometry) > 2;
IF deferred THEN
SELECT bool_or(not (rank_address = 0 or rank_address > 25)) INTO deferred
FROM placex
WHERE osm_type = OLD.osm_type and osm_id = OLD.osm_id
and class = OLD.class and type = OLD.type;
END IF;
-- mark for delete
UPDATE placex set indexed_status = 100 where osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type;
INSERT INTO place_to_be_deleted (osm_type, osm_id, class, type, deferred)
VALUES(OLD.osm_type, OLD.osm_id, OLD.class, OLD.type, deferred);
-- interpolations are special
IF OLD.osm_type='W' and OLD.class = 'place' and OLD.type = 'houses' THEN
UPDATE location_property_osmline set indexed_status = 100 where osm_id = OLD.osm_id; -- osm_id = wayid (=old.osm_id)
END IF;
RETURN OLD;
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION flush_deleted_places()
RETURNS INTEGER
AS $$
BEGIN
-- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
INSERT INTO import_polygon_delete (osm_type, osm_id, class, type)
SELECT osm_type, osm_id, class, type FROM place_to_be_deleted WHERE deferred;
-- delete from place table
ALTER TABLE place DISABLE TRIGGER place_before_delete;
DELETE FROM place USING place_to_be_deleted
WHERE place.osm_type = place_to_be_deleted.osm_type
and place.osm_id = place_to_be_deleted.osm_id
and place.class = place_to_be_deleted.class
and place.type = place_to_be_deleted.type
and not deferred;
ALTER TABLE place ENABLE TRIGGER place_before_delete;
-- Mark for delete in the placex table
UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
WHERE placex.osm_type = place_to_be_deleted.osm_type
and placex.osm_id = place_to_be_deleted.osm_id
and placex.class = place_to_be_deleted.class
and placex.type = place_to_be_deleted.type
and not deferred;
-- Mark for delete in interpolations
UPDATE location_property_osmline SET indexed_status = 100 FROM place_to_be_deleted
WHERE place_to_be_deleted.osm_type = 'W'
and place_to_be_deleted.class = 'place'
and place_to_be_deleted.type = 'houses'
and location_property_osmline.osm_id = place_to_be_deleted.osm_id
and not deferred;
-- Clear todo list.
TRUNCATE TABLE place_to_be_deleted;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;

10
lib-sql/indices.sql
View File

@ -82,4 +82,14 @@ CREATE INDEX IF NOT EXISTS idx_postcode_postcode
INCLUDE (startnumber, endnumber) {{db.tablespace.search_index}}
WHERE startnumber is not null;
{% endif %}
---
-- Table needed for running updates with osm2pgsql on place.
CREATE TABLE IF NOT EXISTS place_to_be_deleted (
osm_type CHAR(1),
osm_id BIGINT,
class TEXT,
type TEXT,
deferred BOOLEAN
);
{% endif %}

1
nominatim/clicmd/args.py
View File

@ -184,6 +184,7 @@ class NominatimArgs:
return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.osm2pgsql_path,
osm2pgsql_cache=self.osm2pgsql_cache or default_cache,
osm2pgsql_style=self.config.get_import_style_file(),
osm2pgsql_style_path=self.config.config_dir,
threads=self.threads or default_threads,
dsn=self.config.get_libpq_dsn(),
flatnode_file=str(self.config.get_path('FLATNODE_FILE') or ''),

10
nominatim/tools/exec_utils.py
View File

@ -10,6 +10,7 @@ Helper functions for executing external programs.
from typing import Any, Union, Optional, Mapping, IO
from pathlib import Path
import logging
import os
import subprocess
import urllib.request as urlrequest
from urllib.parse import urlencode
@ -120,9 +121,16 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
'--log-progress', 'true',
'--number-processes', str(options['threads']),
'--cache', str(options['osm2pgsql_cache']),
'--output', 'gazetteer',
'--style', str(options['osm2pgsql_style'])
]
if str(options['osm2pgsql_style']).endswith('.lua'):
env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / 'flex-base.lua'),
os.environ.get('LUAPATH', ';')))
cmd.extend(('--output', 'flex'))
else:
cmd.extend(('--output', 'gazetteer'))
if options['append']:
cmd.append('--append')
else:

17
nominatim/tools/migration.py
View File

@ -315,3 +315,20 @@ def mark_internal_country_names(conn: Connection, config: Configuration, **_: An
names = {}
names['countrycode'] = country_code
analyzer.add_country_names(country_code, names)
@_migration(4, 1, 99, 0)
def add_place_deletion_todo_table(conn: Connection, **_: Any) -> None:
""" Add helper table for deleting data on updates.
The table is only necessary when updates are possible, i.e.
the database is not in freeze mode.
"""
if conn.table_exists('place'):
with conn.cursor() as cur:
cur.execute("""CREATE TABLE IF NOT EXISTS place_to_be_deleted (
osm_type CHAR(1),
osm_id BIGINT,
class TEXT,
type TEXT,
deferred BOOLEAN)""")

24
nominatim/tools/replication.py
View File

@ -130,10 +130,7 @@ def update(conn: Connection, options: MutableMapping[str, Any],
if endseq is None:
return UpdateState.NO_CHANGES
# Consume updates with osm2pgsql.
options['append'] = True
options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
run_osm2pgsql(options)
run_osm2pgsql_updates(conn, options)
# Write the current status to the file
endstate = repl.get_state_info(endseq)
@ -143,6 +140,25 @@ def update(conn: Connection, options: MutableMapping[str, Any],
return UpdateState.UP_TO_DATE
def run_osm2pgsql_updates(conn: Connection, options: MutableMapping[str, Any]) -> None:
""" Run osm2pgsql in append mode.
"""
# Remove any stale deletion marks.
with conn.cursor() as cur:
cur.execute('TRUNCATE place_to_be_deleted')
conn.commit()
# Consume updates with osm2pgsql.
options['append'] = True
options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
run_osm2pgsql(options)
# Handle deletions
with conn.cursor() as cur:
cur.execute('SELECT flush_deleted_places()')
conn.commit()
def _make_replication_server(url: str, timeout: int) -> ContextManager[ReplicationServer]:
""" Returns a ReplicationServer in form of a context manager.

2
nominatim/version.py
View File

@ -25,7 +25,7 @@ from typing import Optional, Tuple
# patch level when cherry-picking the commit with the migration.
#
# Released versions always have a database patch level of 0.
NOMINATIM_VERSION = (4, 1, 0, 0)
NOMINATIM_VERSION = (4, 1, 99, 0)
POSTGRESQL_REQUIRED_VERSION = (9, 6)
POSTGIS_REQUIRED_VERSION = (2, 2)

393
settings/flex-base.lua Normal file
View File

@ -0,0 +1,393 @@
-- Core functions for Nominatim import flex style.
--
-- The single place table.
place_table = osm2pgsql.define_table{
name = "place",
ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
columns = {
{ column = 'class', type = 'text', not_null = true },
{ column = 'type', type = 'text', not_null = true },
{ column = 'admin_level', type = 'smallint' },
{ column = 'name', type = 'hstore' },
{ column = 'address', type = 'hstore' },
{ column = 'extratags', type = 'hstore' },
{ column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
}
}
------------- Place class ------------------------------------------
local Place = {}
Place.__index = Place
function Place.new(object, geom_func)
local self = setmetatable({}, Place)
self.object = object
self.geom_func = geom_func
self.admin_level = tonumber(self.object:grab_tag('admin_level'))
if self.admin_level == nil
or self.admin_level <= 0 or self.admin_level > 15
or math.floor(self.admin_level) ~= self.admin_level then
self.admin_level = 15
end
self.num_entries = 0
self.has_name = false
self.names = {}
self.address = {}
self.extratags = {}
return self
end
function Place:delete(data)
if data.match ~= nil then
for k, v in pairs(self.object.tags) do
if data.match(k, v) then
self.object.tags[k] = nil
end
end
end
end
function Place:grab_extratags(data)
local count = 0
if data.match ~= nil then
for k, v in pairs(self.object.tags) do
if data.match(k, v) then
self.object.tags[k] = nil
self.extratags[k] = v
count = count + 1
end
end
end
return count
end
function Place:grab_address(data)
local count = 0
if data.match ~= nil then
for k, v in pairs(self.object.tags) do
if data.match(k, v) then
self.object.tags[k] = nil
if data.include_on_name == true then
self.has_name = true
end
if data.out_key ~= nil then
self.address[data.out_key] = v
return 1
end
if k:sub(1, 5) == 'addr:' then
self.address[k:sub(6)] = v
elseif k:sub(1, 6) == 'is_in:' then
self.address[k:sub(7)] = v
else
self.address[k] = v
end
count = count + 1
end
end
end
return count
end
function Place:set_address(key, value)
self.address[key] = value
end
function Place:grab_name(data)
local count = 0
if data.match ~= nil then
for k, v in pairs(self.object.tags) do
if data.match(k, v) then
self.object.tags[k] = nil
self.names[k] = v
if data.include_on_name ~= false then
self.has_name = true
end
count = count + 1
end
end
end
return count
end
function Place:grab_tag(key)
return self.object:grab_tag(key)
end
function Place:tags()
return self.object.tags
end
function Place:write_place(k, v, mtype, save_extra_mains)
if mtype == nil then
return 0
end
v = v or self.object.tags[k]
if v == nil then
return 0
end
if type(mtype) == 'table' then
mtype = mtype[v] or mtype[1]
end
if mtype == 'always' or (self.has_name and mtype == 'named') then
return self:write_row(k, v, save_extra_mains)
end
if mtype == 'named_with_key' then
local names = {}
local prefix = k .. ':name'
for namek, namev in pairs(self.object.tags) do
if namek:sub(1, #prefix) == prefix
and (#namek == #prefix
or namek:sub(#prefix + 1, #prefix + 1) == ':') then
names[namek:sub(#k + 2)] = namev
end
end
if next(names) ~= nil then
local saved_names = self.names
self.names = names
local results = self:write_row(k, v, save_extra_mains)
self.names = saved_names
return results
end
end
return 0
end
function Place:write_row(k, v, save_extra_mains)
if self.geometry == nil then
self.geometry = self.geom_func(self.object)
end
if self.geometry:is_null() then
return 0
end
if save_extra_mains then
for extra_k, extra_v in pairs(self.object.tags) do
if extra_k ~= k then
self.extratags[extra_k] = extra_v
end
end
end
place_table:insert{
class = k,
type = v,
admin_level = self.admin_level,
name = next(self.names) and self.names,
address = next(self.address) and self.address,
extratags = next(self.extratags) and self.extratags,
geometry = self.geometry
}
if save_extra_mains then
for k, v in pairs(self.object.tags) do
self.extratags[k] = nil
end
end
self.num_entries = self.num_entries + 1
return 1
end
function tag_match(data)
if data == nil or next(data) == nil then
return nil
end
local fullmatches = {}
local key_prefixes = {}
local key_suffixes = {}
if data.keys ~= nil then
for _, key in pairs(data.keys) do
if key:sub(1, 1) == '*' then
if #key > 1 then
if key_suffixes[#key - 1] == nil then
key_suffixes[#key - 1] = {}
end
key_suffixes[#key - 1][key:sub(2)] = true
end
elseif key:sub(#key, #key) == '*' then
if key_prefixes[#key - 1] == nil then
key_prefixes[#key - 1] = {}
end
key_prefixes[#key - 1][key:sub(1, #key - 1)] = true
else
fullmatches[key] = true
end
end
end
if data.tags ~= nil then
for k, vlist in pairs(data.tags) do
if fullmatches[k] == nil then
fullmatches[k] = {}
for _, v in pairs(vlist) do
fullmatches[k][v] = true
end
end
end
end
return function (k, v)
if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then
return true
end
for slen, slist in pairs(key_suffixes) do
if #k >= slen and slist[k:sub(-slen)] ~= nil then
return true
end
end
for slen, slist in pairs(key_prefixes) do
if #k >= slen and slist[k:sub(1, slen)] ~= nil then
return true
end
end
return false
end
end
-- Process functions for all data types
function osm2pgsql.process_node(object)
local function geom_func(o)
return o:as_point()
end
process_tags(Place.new(object, geom_func))
end
function osm2pgsql.process_way(object)
local function geom_func(o)
local geom = o:as_polygon()
if geom:is_null() then
geom = o:as_linestring()
end
return geom
end
process_tags(Place.new(object, geom_func))
end
function relation_as_multipolygon(o)
return o:as_multipolygon()
end
function relation_as_multiline(o)
return o:as_multilinestring():line_merge()
end
function osm2pgsql.process_relation(object)
local geom_func = RELATION_TYPES[object.tags.type]
if geom_func ~= nil then
process_tags(Place.new(object, geom_func))
end
end
function process_tags(o)
local fallback
o:delete{match = PRE_DELETE}
o:grab_extratags{match = PRE_EXTRAS}
-- Exception for boundary/place double tagging
if o.object.tags.boundary == 'administrative' then
o:grab_extratags{match = function (k, v)
return k == 'place' and v:sub(1,3) ~= 'isl'
end}
end
-- address keys
o:grab_address{match=COUNTRY_TAGS, out_key='country'}
if o.address.country ~= nil and #o.address.country ~= 2 then
o.address['country'] = nil
end
if o:grab_name{match=HOUSENAME_TAGS} > 0 then
fallback = {'place', 'house'}
end
if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then
fallback = {'place', 'house'}
end
if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then
fallback = {'place', 'postcode'}
end
local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0
if ADD_TIGER_COUNTY then
local v = o:grab_tag('tiger:county')
if v ~= nil then
v, num = v:gsub(',.*', ' county')
if num == 0 then
v = v .. ' county'
end
o:set_address('tiger:county', v)
end
end
o:grab_address{match=ADDRESS_TAGS}
if is_interpolation then
o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
return
end
-- name keys
o:grab_name{match = NAMES}
o:grab_name{match = REFS, include_on_name = false}
o:delete{match = POST_DELETE}
o:grab_extratags{match = POST_EXTRAS}
-- collect main keys
local num_mains = 0
for k, v in pairs(o:tags()) do
num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
end
if num_mains == 0 then
for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do
if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then
return
end
end
if fallback ~= nil then
o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS)
end
end
end

130
settings/import-extratags.lua Normal file
View File

@ -0,0 +1,130 @@
require('flex-base')
RELATION_TYPES = {
multipolygon = relation_as_multipolygon,
boundary = relation_as_multipolygon,
waterway = relation_as_multiline
}
MAIN_KEYS = {
emergency = 'always',
historic = 'always',
military = 'always',
natural = 'named',
landuse = 'named',
highway = {'always',
street_lamp = 'named',
traffic_signals = 'named',
service = 'named',
cycleway = 'named',
path = 'named',
footway = 'named',
steps = 'named',
bridleway = 'named',
track = 'named',
motorway_link = 'named',
trunk_link = 'named',
primary_link = 'named',
secondary_link = 'named',
tertiary_link = 'named'},
railway = 'named',
man_made = 'always',
aerialway = 'always',
boundary = {'named',
postal_code = 'named'},
aeroway = 'always',
amenity = 'always',
club = 'always',
craft = 'always',
leisure = 'always',
office = 'always',
mountain_pass = 'always',
shop = 'always',
tourism = 'always',
bridge = 'named_with_key',
tunnel = 'named_with_key',
waterway = 'named',
place = 'always'
}
MAIN_FALLBACK_KEYS = {
building = 'named',
landuse = 'named',
junction = 'named',
healthcare = 'named'
}
PRE_DELETE = tag_match{keys = {'note', 'note:*', 'source', 'source*', 'attribution',
'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
'ref:linz:*', 'is_in:postcode'},
tags = {emergency = {'yes', 'no', 'fire_hydrant'},
historic = {'yes', 'no'},
military = {'yes', 'no'},
natural = {'yes', 'no', 'coastline'},
highway = {'no', 'turning_circle', 'mini_roundabout',
'noexit', 'crossing', 'give_way', 'stop'},
railway = {'level_crossing', 'no', 'rail'},
man_made = {'survey_point', 'cutline'},
aerialway = {'pylon', 'no'},
aeroway = {'no'},
amenity = {'no'},
club = {'no'},
craft = {'no'},
leisure = {'no'},
office = {'no'},
mountain_pass = {'no'},
shop = {'no'},
tourism = {'yes', 'no'},
bridge = {'no'},
tunnel = {'no'},
waterway = {'riverbank'},
building = {'no'},
boundary = {'place'}}
}
POST_DELETE = tag_match{keys = {'tiger:*'}}
PRE_EXTRAS = tag_match{keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
'name:etymology', 'name:signed', 'name:botanical',
'wikidata', '*:wikidata',
'addr:street:name', 'addr:street:type'}
}
NAMES = tag_match{keys = {'name', 'name:*',
'int_name', 'int_name:*',
'nat_name', 'nat_name:*',
'reg_name', 'reg_name:*',
'loc_name', 'loc_name:*',
'old_name', 'old_name:*',
'alt_name', 'alt_name:*', 'alt_name_*',
'official_name', 'official_name:*',
'place_name', 'place_name:*',
'short_name', 'short_name:*', 'brand'}}
REFS = tag_match{keys = {'ref', 'int_ref', 'nat_ref', 'reg_ref', 'loc_ref', 'old_ref',
'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}}
POSTCODES = tag_match{keys = {'postal_code', 'postcode', 'addr:postcode',
'tiger:zip_left', 'tiger:zip_right'}}
COUNTRY_TAGS = tag_match{keys = {'country_code', 'ISO3166-1',
'addr:country_code', 'is_in:country_code',
'addr:country', 'is_in:country'}}
HOUSENAME_TAGS = tag_match{keys = {'addr:housename'}}
HOUSENUMBER_TAGS = tag_match{keys = {'addr:housenumber', 'addr:conscriptionnumber',
'addr:streetnumber'}}
INTERPOLATION_TAGS = tag_match{keys = {'addr:interpolation'}}
ADDRESS_TAGS = tag_match{keys = {'addr:*', 'is_in:*'}}
ADD_TIGER_COUNTY = true
SAVE_EXTRA_MAINS = true

1
test/bdd/environment.py
View File

@ -27,6 +27,7 @@ userconfig = {
'API_TEST_FILE' : (TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf').resolve(),
'SERVER_MODULE_PATH' : None,
'TOKENIZER' : None, # Test with a custom tokenizer
'STYLE' : 'extratags',
'PHPCOV' : False, # set to output directory to enable code coverage
}

209
test/bdd/osm2pgsql/import/tags.feature Normal file
View File

@ -0,0 +1,209 @@
@DB
Feature: Tag evaluation
Tests if tags are correctly imported into the place table
Scenario: Main tags as fallback
When loading osm data
"""
n100 Tjunction=yes,highway=bus_stop
n101 Tjunction=yes,name=Bar
n200 Tbuilding=yes,amenity=cafe
n201 Tbuilding=yes,name=Intersting
n202 Tbuilding=yes
"""
Then place contains exactly
| object | class | type |
| N100 | highway | bus_stop |
| N101 | junction | yes |
| N200 | amenity | cafe |
| N201 | building | yes |
Scenario: Name and reg tags
When loading osm data
"""
n2001 Thighway=road,name=Foo,alt_name:de=Bar,ref=45
n2002 Thighway=road,name:prefix=Pre,name:suffix=Post,ref:de=55
n2003 Thighway=yes,name:%20%de=Foo,name=real1
n2004 Thighway=yes,name:%a%de=Foo,name=real2
n2005 Thighway=yes,name:%9%de=Foo,name:\\=real3
n2006 Thighway=yes,name:%9%de=Foo,name=rea\l3
"""
Then place contains exactly
| object | class | type | name |
| N2001 | highway | road | 'name': 'Foo', 'alt_name:de': 'Bar', 'ref': '45' |
| N2002 | highway | road | - |
| N2003 | highway | yes | 'name: de': 'Foo', 'name': 'real1' |
| N2004 | highway | yes | 'name:\nde': 'Foo', 'name': 'real2' |
| N2005 | highway | yes | 'name:\tde': 'Foo', 'name:\\\\': 'real3' |
| N2006 | highway | yes | 'name:\tde': 'Foo', 'name': 'rea\\l3' |
And place contains
| object | extratags |
| N2002 | 'name:prefix': 'Pre', 'name:suffix': 'Post', 'ref:de': '55' |
Scenario: Name when using with_name flag
When loading osm data
"""
n3001 Tbridge=yes,bridge:name=GoldenGate
n3002 Tbridge=yes,bridge:name:en=Rainbow
"""
Then place contains exactly
| object | class | type | name |
| N3001 | bridge | yes | 'name': 'GoldenGate' |
| N3002 | bridge | yes | 'name:en': 'Rainbow' |
Scenario: Address tags
When loading osm data
"""
n4001 Taddr:housenumber=34,addr:city=Esmarald,addr:county=Land
n4002 Taddr:streetnumber=10,is_in:city=Rootoo,is_in=Gold
"""
Then place contains exactly
| object | class | address |
| N4001 | place | 'housenumber': '34', 'city': 'Esmarald', 'county': 'Land' |
| N4002 | place | 'streetnumber': '10', 'city': 'Rootoo' |
Scenario: Country codes
When loading osm data
"""
n5001 Tshop=yes,country_code=DE
n5002 Tshop=yes,country_code=toolong
n5003 Tshop=yes,country_code=x
n5004 Tshop=yes,addr:country=us
n5005 Tshop=yes,country=be
n5006 Tshop=yes,addr:country=France
"""
Then place contains exactly
| object | class | address |
| N5001 | shop | 'country': 'DE' |
| N5002 | shop | - |
| N5003 | shop | - |
| N5004 | shop | 'country': 'us' |
| N5005 | shop | - |
| N5006 | shop | - |
Scenario: Postcodes
When loading osm data
"""
n6001 Tshop=bank,addr:postcode=12345
n6002 Tshop=bank,tiger:zip_left=34343
n6003 Tshop=bank,is_in:postcode=9009
"""
Then place contains exactly
| object | class | address |
| N6001 | shop | 'postcode': '12345' |
| N6002 | shop | 'postcode': '34343' |
| N6003 | shop | - |
Scenario: Main with extra
When loading osm data
"""
n7001 Thighway=primary,bridge=yes,name=1
n7002 Thighway=primary,bridge=yes,bridge:name=1
"""
Then place contains exactly
| object | class | type | name | extratags+bridge:name |
| N7001 | highway | primary | 'name': '1' | - |
| N7002:highway | highway | primary | - | 1 |
| N7002:bridge | bridge | yes | 'name': '1' | 1 |
Scenario: Global fallback and skipping
When loading osm data
"""
n8001 Tshop=shoes,note:de=Nein,xx=yy
n8002 Tshop=shoes,building=no,ele=234
n8003 Tshop=shoes,name:source=survey
"""
Then place contains exactly
| object | class | extratags |
| N8001 | shop | 'xx': 'yy' |
| N8002 | shop | 'ele': '234' |
| N8003 | shop | - |
Scenario: Admin levels
When loading osm data
"""
n9001 Tplace=city
n9002 Tplace=city,admin_level=16
n9003 Tplace=city,admin_level=x
n9004 Tplace=city,admin_level=1
n9005 Tplace=city,admin_level=0
n9006 Tplace=city,admin_level=2.5
"""
Then place contains exactly
| object | class | admin_level |
| N9001 | place | 15 |
| N9002 | place | 15 |
| N9003 | place | 15 |
| N9004 | place | 1 |
| N9005 | place | 15 |
| N9006 | place | 15 |
Scenario: Administrative boundaries with place tags
When loading osm data
"""
n10001 Tboundary=administrative,place=city,name=A
n10002 Tboundary=natural,place=city,name=B
n10003 Tboundary=administrative,place=island,name=C
"""
Then place contains
| object | class | type | extratags |
| N10001 | boundary | administrative | 'place': 'city' |
And place contains
| object | class | type |
| N10002:boundary | boundary | natural |
| N10002:place | place | city |
| N10003:boundary | boundary | administrative |
| N10003:place | place | island |
Scenario: Shorten tiger:county tags
When loading osm data
"""
n11001 Tplace=village,tiger:county=Feebourgh%2c%%20%AL
n11002 Tplace=village,addr:state=Alabama,tiger:county=Feebourgh%2c%%20%AL
n11003 Tplace=village,tiger:county=Feebourgh
"""
Then place contains exactly
| object | class | address |
| N11001 | place | 'tiger:county': 'Feebourgh county' |
| N11002 | place | 'tiger:county': 'Feebourgh county', 'state': 'Alabama' |
| N11003 | place | 'tiger:county': 'Feebourgh county' |
Scenario: Building fallbacks
When loading osm data
"""
n12001 Ttourism=hotel,building=yes
n12002 Tbuilding=house
n12003 Tbuilding=shed,addr:housenumber=1
n12004 Tbuilding=yes,name=Das-Haus
n12005 Tbuilding=yes,addr:postcode=12345
"""
Then place contains exactly
| object | class | type |
| N12001 | tourism | hotel |
| N12003 | building | shed |
| N12004 | building | yes |
| N12005 | place | postcode |
Scenario: Address interpolations
When loading osm data
"""
n13001 Taddr:interpolation=odd
n13002 Taddr:interpolation=even,place=city
"""
Then place contains exactly
| object | class | type | address |
| N13001 | place | houses | 'interpolation': 'odd' |
| N13002 | place | houses | 'interpolation': 'even' |

133
test/bdd/osm2pgsql/update/interpolations.feature Normal file
View File

@ -0,0 +1,133 @@
@DB
Feature: Updates of address interpolation objects
Test that changes to address interpolation objects are correctly
propagated.
Background:
Given the grid
| 1 | 2 |
Scenario: Adding a new interpolation
When loading osm data
"""
n1 Taddr:housenumber=3
n2 Taddr:housenumber=17
"""
Then place contains
| object | type |
| N1:place | house |
| N2:place | house |
When updating osm data
"""
w99 Taddr:interpolation=odd Nn1,n2
"""
Then place contains
| object | type |
| N1:place | house |
| N2:place | house |
| W99:place | houses |
When indexing
Then placex contains exactly
| object | type |
| N1:place | house |
| N2:place | house |
Then location_property_osmline contains exactly
| object |
| 99:5 |
Scenario: Delete an existing interpolation
When loading osm data
"""
n1 Taddr:housenumber=2
n2 Taddr:housenumber=7
w99 Taddr:interpolation=odd Nn1,n2
"""
Then place contains
| object | type |
| N1:place | house |
| N2:place | house |
| W99:place | houses |
When updating osm data
"""
w99 v2 dD
"""
Then place contains
| object | type |
| N1:place | house |
| N2:place | house |
When indexing
Then placex contains exactly
| object | type |
| N1:place | house |
| N2:place | house |
Then location_property_osmline contains exactly
| object | indexed_status |
Scenario: Changing an object to an interpolation
When loading osm data
"""
n1 Taddr:housenumber=3
n2 Taddr:housenumber=17
w99 Thighway=residential Nn1,n2
"""
Then place contains
| object | type |
| N1:place | house |
| N2:place | house |
| W99:highway | residential |
When updating osm data
"""
w99 Taddr:interpolation=odd Nn1,n2
"""
Then place contains
| object | type |
| N1:place | house |
| N2:place | house |
| W99:place | houses |
When indexing
Then placex contains exactly
| object | type |
| N1:place | house |
| N2:place | house |
And location_property_osmline contains exactly
| object |
| 99:5 |
Scenario: Changing an interpolation to something else
When loading osm data
"""
n1 Taddr:housenumber=3
n2 Taddr:housenumber=17
w99 Taddr:interpolation=odd Nn1,n2
"""
Then place contains
| object | type |
| N1:place | house |
| N2:place | house |
| W99:place | houses |
When updating osm data
"""
w99 Thighway=residential Nn1,n2
"""
Then place contains
| object | type |
| N1:place | house |
| N2:place | house |
| W99:highway | residential |
When indexing
Then placex contains exactly
| object | type |
| N1:place | house |
| N2:place | house |
| W99:highway | residential |
And location_property_osmline contains exactly
| object |

163
test/bdd/osm2pgsql/update/postcodes.feature Normal file
View File

@ -0,0 +1,163 @@
@DB
Feature: Update of postcode only objects
Tests that changes to objects containing only a postcode are
propagated correctly.
Scenario: Adding a postcode-only node
When loading osm data
"""
"""
Then place contains exactly
| object |
When updating osm data
"""
n34 Tpostcode=4456
"""
Then place contains exactly
| object | type |
| N34:place | postcode |
When indexing
Then placex contains exactly
| object |
Scenario: Deleting a postcode-only node
When loading osm data
"""
n34 Tpostcode=4456
"""
Then place contains exactly
| object | type |
| N34:place | postcode |
When updating osm data
"""
n34 v2 dD
"""
Then place contains exactly
| object |
When indexing
Then placex contains exactly
| object |
Scenario Outline: Converting a regular object into a postcode-only node
When loading osm data
"""
n34 T<class>=<type>
"""
Then place contains exactly
| object | type |
| N34:<class> | <type> |
When updating osm data
"""
n34 Tpostcode=4456
"""
Then place contains exactly
| object | type |
| N34:place | postcode |
When indexing
Then placex contains exactly
| object |
Examples:
| class | type |
| amenity | restaurant |
| place | hamlet |
Scenario Outline: Converting a postcode-only node into a regular object
When loading osm data
"""
n34 Tpostcode=4456
"""
Then place contains exactly
| object | type |
| N34:place | postcode |
When updating osm data
"""
n34 T<class>=<type>
"""
Then place contains exactly
| object | type |
| N34:<class> | <type> |
When indexing
Then placex contains exactly
| object | type |
| N34:<class> | <type> |
Examples:
| class | type |
| amenity | restaurant |
| place | hamlet |
Scenario: Converting na interpolation into a postcode-only node
Given the grid
| 1 | 2 |
When loading osm data
"""
n1 Taddr:housenumber=3
n2 Taddr:housenumber=17
w34 Taddr:interpolation=odd Nn1,n2
"""
Then place contains exactly
| object | type |
| N1:place | house |
| N2:place | house |
| W34:place | houses |
When updating osm data
"""
w34 Tpostcode=4456 Nn1,n2
"""
Then place contains exactly
| object | type |
| N1:place | house |
| N2:place | house |
| W34:place | postcode |
When indexing
Then location_property_osmline contains exactly
| object |
And placex contains exactly
| object | type |
| N1:place | house |
| N2:place | house |
Scenario: Converting a postcode-only node into an interpolation
Given the grid
| 1 | 2 |
When loading osm data
"""
n1 Taddr:housenumber=3
n2 Taddr:housenumber=17
w34 Tpostcode=4456 Nn1,n2
"""
Then place contains exactly
| object | type |
| N1:place | house |
| N2:place | house |
| W34:place | postcode |
When updating osm data
"""
w34 Taddr:interpolation=odd Nn1,n2
"""
Then place contains exactly
| object | type |
| N1:place | house |
| N2:place | house |
| W34:place | houses |
When indexing
Then location_property_osmline contains exactly
| object |
| 34:5 |
And placex contains exactly
| object | type |
| N1:place | house |
| N2:place | house |

86
test/bdd/osm2pgsql/update/simple.feature
View File

@ -2,60 +2,48 @@
Feature: Update of simple objects by osm2pgsql
Testing basic update functions of osm2pgsql.
Scenario: Import object with two main tags
Scenario: Adding a new object
When loading osm data
"""
n1 Ttourism=hotel,amenity=restaurant,name=foo
n2 Tplace=locality,name=spotty
n1 Tplace=town,name=Middletown
"""
Then place contains
| object | type | name+name |
| N1:tourism | hotel | foo |
| N1:amenity | restaurant | foo |
| N2:place | locality | spotty |
When updating osm data
"""
n1 dV Ttourism=hotel,name=foo
n2 dD
"""
Then place has no entry for N1:amenity
And place has no entry for N2
And place contains
| object | class | type | name |
| N1:tourism | tourism | hotel | 'name' : 'foo' |
Then place contains exactly
| object | type | name+name |
| N1:place | town | Middletown |
Scenario: Downgrading a highway to one that is dropped without name
When loading osm data
"""
n100 x0 y0
n101 x0.0001 y0.0001
w1 Thighway=residential Nn100,n101
"""
Then place contains
| object |
| W1:highway |
When updating osm data
"""
w1 Thighway=service Nn100,n101
"""
Then place has no entry for W1
When updating osm data
"""
n2 Tamenity=hotel,name=Posthotel
"""
Then place contains exactly
| object | type | name+name |
| N1:place | town | Middletown |
| N2:amenity | hotel | Posthotel |
And placex contains exactly
| object | type | name+name | indexed_status |
| N1:place | town | Middletown | 0 |
| N2:amenity | hotel | Posthotel | 1 |
Scenario: Downgrading a highway when a second tag is present
Scenario: Deleting an existing object
When loading osm data
"""
n100 x0 y0
n101 x0.0001 y0.0001
w1 Thighway=residential,tourism=hotel Nn100,n101
n1 Tplace=town,name=Middletown
n2 Tamenity=hotel,name=Posthotel
"""
Then place contains
| object |
| W1:highway |
| W1:tourism |
When updating osm data
"""
w1 Thighway=service,tourism=hotel Nn100,n101
"""
Then place has no entry for W1:highway
And place contains
| object |
| W1:tourism |
Then place contains exactly
| object | type | name+name |
| N1:place | town | Middletown |
| N2:amenity | hotel | Posthotel |
When updating osm data
"""
n2 dD
"""
Then place contains exactly
| object | type | name+name |
| N1:place | town | Middletown |
And placex contains exactly
| object | type | name+name | indexed_status |
| N1:place | town | Middletown | 0 |
| N2:amenity | hotel | Posthotel | 100 |

490
test/bdd/osm2pgsql/update/tags.feature Normal file
View File

@ -0,0 +1,490 @@
@DB
Feature: Tag evaluation
Tests if tags are correctly updated in the place table
Background:
Given the grid
| 1 | 2 | 3 |
| 10 | 11 | |
| 45 | 46 | |
Scenario: Main tag deleted
When loading osm data
"""
n1 Tamenity=restaurant
n2 Thighway=bus_stop,railway=stop,name=X
n3 Tamenity=prison
"""
Then place contains exactly
| object | class | type |
| N1 | amenity | restaurant |
| N2:highway | highway | bus_stop |
| N2:railway | railway | stop |
| N3 | amenity | prison |
When updating osm data
"""
n1 Tnot_a=restaurant
n2 Thighway=bus_stop,name=X
"""
Then place contains exactly
| object | class | type |
| N2:highway | highway | bus_stop |
| N3 | amenity | prison |
And placex contains
| object | indexed_status |
| N3:amenity | 0 |
When indexing
Then placex contains exactly
| object | type | name |
| N2:highway | bus_stop | 'name': 'X' |
| N3:amenity | prison | - |
Scenario: Main tag added
When loading osm data
"""
n1 Tatity=restaurant
n2 Thighway=bus_stop,name=X
"""
Then place contains exactly
| object | class | type |
| N2:highway | highway | bus_stop |
When updating osm data
"""
n1 Tamenity=restaurant
n2 Thighway=bus_stop,railway=stop,name=X
"""
Then place contains exactly
| object | class | type |
| N1 | amenity | restaurant |
| N2:highway | highway | bus_stop |
| N2:railway | railway | stop |
When indexing
Then placex contains exactly
| object | type | name |
| N1:amenity | restaurant | - |
| N2:highway | bus_stop | 'name': 'X' |
| N2:railway | stop | 'name': 'X' |
Scenario: Main tag modified
When loading osm data
"""
n10 Thighway=footway,name=X
n11 Tamenity=atm
"""
Then place contains exactly
| object | class | type |
| N10 | highway | footway |
| N11 | amenity | atm |
When updating osm data
"""
n10 Thighway=path,name=X
n11 Thighway=primary
"""
Then place contains exactly
| object | class | type |
| N10 | highway | path |
| N11 | highway | primary |
When indexing
Then placex contains exactly
| object | type | name |
| N10:highway | path | 'name': 'X' |
| N11:highway | primary | - |
Scenario: Main tags with name, name added
When loading osm data
"""
n45 Tlanduse=cemetry
n46 Tbuilding=yes
"""
Then place contains exactly
| object | class | type |
When updating osm data
"""
n45 Tlanduse=cemetry,name=TODO
n46 Tbuilding=yes,addr:housenumber=1
"""
Then place contains exactly
| object | class | type |
| N45 | landuse | cemetry |
| N46 | building| yes |
When indexing
Then placex contains exactly
| object | type | name | address |
| N45:landuse | cemetry | 'name': 'TODO' | - |
| N46:building| yes | - | 'housenumber': '1' |
Scenario: Main tags with name, name removed
When loading osm data
"""
n45 Tlanduse=cemetry,name=TODO
n46 Tbuilding=yes,addr:housenumber=1
"""
Then place contains exactly
| object | class | type |
| N45 | landuse | cemetry |
| N46 | building| yes |
When updating osm data
"""
n45 Tlanduse=cemetry
n46 Tbuilding=yes
"""
Then place contains exactly
| object | class | type |
When indexing
Then placex contains exactly
| object |
Scenario: Main tags with name, name modified
When loading osm data
"""
n45 Tlanduse=cemetry,name=TODO
n46 Tbuilding=yes,addr:housenumber=1
"""
Then place contains exactly
| object | class | type | name | address |
| N45 | landuse | cemetry | 'name' : 'TODO' | - |
| N46 | building| yes | - | 'housenumber': '1'|
When updating osm data
"""
n45 Tlanduse=cemetry,name=DONE
n46 Tbuilding=yes,addr:housenumber=10
"""
Then place contains exactly
| object | class | type | name | address |
| N45 | landuse | cemetry | 'name' : 'DONE' | - |
| N46 | building| yes | - | 'housenumber': '10'|
When indexing
Then placex contains exactly
| object | class | type | name | address |
| N45 | landuse | cemetry | 'name' : 'DONE' | - |
| N46 | building| yes | - | 'housenumber': '10'|
Scenario: Main tag added to address only node
When loading osm data
"""
n1 Taddr:housenumber=345
"""
Then place contains exactly
| object | class | type | address |
| N1 | place | house | 'housenumber': '345'|
When updating osm data
"""
n1 Taddr:housenumber=345,building=yes
"""
Then place contains exactly
| object | class | type | address |
| N1 | building | yes | 'housenumber': '345'|
When indexing
Then placex contains exactly
| object | class | type | address |
| N1 | building | yes | 'housenumber': '345'|
Scenario: Main tag removed from address only node
When loading osm data
"""
n1 Taddr:housenumber=345,building=yes
"""
Then place contains exactly
| object | class | type | address |
| N1 | building | yes | 'housenumber': '345'|
When updating osm data
"""
n1 Taddr:housenumber=345
"""
Then place contains exactly
| object | class | type | address |
| N1 | place | house | 'housenumber': '345'|
When indexing
Then placex contains exactly
| object | class | type | address |
| N1 | place | house | 'housenumber': '345'|
Scenario: Main tags with name key, adding key name
When loading osm data
"""
n2 Tbridge=yes
"""
Then place contains exactly
| object | class | type |
When updating osm data
"""
n2 Tbridge=yes,bridge:name=high
"""
Then place contains exactly
| object | class | type | name |
| N2 | bridge | yes | 'name': 'high' |
When indexing
Then placex contains exactly
| object | class | type | name |
| N2 | bridge | yes | 'name': 'high' |
Scenario: Main tags with name key, deleting key name
When loading osm data
"""
n2 Tbridge=yes,bridge:name=high
"""
Then place contains exactly
| object | class | type | name |
| N2 | bridge | yes | 'name': 'high' |
When updating osm data
"""
n2 Tbridge=yes
"""
Then place contains exactly
| object |
When indexing
Then placex contains exactly
| object |
Scenario: Main tags with name key, changing key name
When loading osm data
"""
n2 Tbridge=yes,bridge:name=high
"""
Then place contains exactly
| object | class | type | name |
| N2 | bridge | yes | 'name': 'high' |
When updating osm data
"""
n2 Tbridge=yes,bridge:name:en=high
"""
Then place contains exactly
| object | class | type | name |
| N2 | bridge | yes | 'name:en': 'high' |
When indexing
Then placex contains exactly
| object | class | type | name |
| N2 | bridge | yes | 'name:en': 'high' |
Scenario: Downgrading a highway to one that is dropped without name
When loading osm data
"""
n100 x0 y0
n101 x0.0001 y0.0001
w1 Thighway=residential Nn100,n101
"""
Then place contains exactly
| object |
| W1:highway |
When updating osm data
"""
w1 Thighway=service Nn100,n101
"""
Then place contains exactly
| object |
When indexing
Then placex contains exactly
| object |
Scenario: Upgrading a highway to one that is not dropped without name
When loading osm data
"""
n100 x0 y0
n101 x0.0001 y0.0001
w1 Thighway=service Nn100,n101
"""
Then place contains exactly
| object |
When updating osm data
"""
w1 Thighway=unclassified Nn100,n101
"""
Then place contains exactly
| object |
| W1:highway |
When indexing
Then placex contains exactly
| object |
| W1:highway |
Scenario: Downgrading a highway when a second tag is present
When loading osm data
"""
n100 x0 y0
n101 x0.0001 y0.0001
w1 Thighway=residential,tourism=hotel Nn100,n101
"""
Then place contains exactly
| object | type |
| W1:highway | residential |
| W1:tourism | hotel |
When updating osm data
"""
w1 Thighway=service,tourism=hotel Nn100,n101
"""
Then place contains exactly
| object | type |
| W1:tourism | hotel |
When indexing
Then placex contains exactly
| object | type |
| W1:tourism | hotel |
Scenario: Upgrading a highway when a second tag is present
When loading osm data
"""
n100 x0 y0
n101 x0.0001 y0.0001
w1 Thighway=service,tourism=hotel Nn100,n101
"""
Then place contains exactly
| object | type |
| W1:tourism | hotel |
When updating osm data
"""
w1 Thighway=residential,tourism=hotel Nn100,n101
"""
Then place contains exactly
| object | type |
| W1:highway | residential |
| W1:tourism | hotel |
When indexing
Then placex contains exactly
| object | type |
| W1:highway | residential |
| W1:tourism | hotel |
Scenario: Replay on administrative boundary
When loading osm data
"""
n10 x34.0 y-4.23
n11 x34.1 y-4.23
n12 x34.2 y-4.13
w10 Tboundary=administrative,waterway=river,name=Border,admin_level=2 Nn12,n11,n10
"""
Then place contains exactly
| object | type | admin_level | name |
| W10:waterway | river | 2 | 'name': 'Border' |
| W10:boundary | administrative | 2 | 'name': 'Border' |
When updating osm data
"""
w10 Tboundary=administrative,waterway=river,name=Border,admin_level=2 Nn12,n11,n10
"""
Then place contains exactly
| object | type | admin_level | name |
| W10:waterway | river | 2 | 'name': 'Border' |
| W10:boundary | administrative | 2 | 'name': 'Border' |
When indexing
Then placex contains exactly
| object | type | admin_level | name |
| W10:waterway | river | 2 | 'name': 'Border' |
Scenario: Change admin_level on administrative boundary
Given the grid
| 10 | 11 |
| 13 | 12 |
When loading osm data
"""
n10
n11
n12
n13
w10 Nn10,n11,n12,n13,n10
r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=2 Mw10@
"""
Then place contains exactly
| object | admin_level |
| R10:boundary | 2 |
When updating osm data
"""
r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
"""
Then place contains exactly
| object | type | admin_level |
| R10:boundary | administrative | 4 |
When indexing
Then placex contains exactly
| object | type | admin_level |
| R10:boundary | administrative | 4 |
Scenario: Change boundary to administrative
Given the grid
| 10 | 11 |
| 13 | 12 |
When loading osm data
"""
n10
n11
n12
n13
w10 Nn10,n11,n12,n13,n10
r10 Ttype=multipolygon,boundary=informal,name=Border,admin_level=4 Mw10@
"""
Then place contains exactly
| object | type | admin_level |
| R10:boundary | informal | 4 |
When updating osm data
"""
r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
"""
Then place contains exactly
| object | type | admin_level |
| R10:boundary | administrative | 4 |
When indexing
Then placex contains exactly
| object | type | admin_level |
| R10:boundary | administrative | 4 |
Scenario: Change boundary away from administrative
Given the grid
| 10 | 11 |
| 13 | 12 |
When loading osm data
"""
n10
n11
n12
n13
w10 Nn10,n11,n12,n13,n10
r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
"""
Then place contains exactly
| object | type | admin_level |
| R10:boundary | administrative | 4 |
When updating osm data
"""
r10 Ttype=multipolygon,boundary=informal,name=Border,admin_level=4 Mw10@
"""
Then place contains exactly
| object | type | admin_level |
| R10:boundary | informal | 4 |
When indexing
Then placex contains exactly
| object | type | admin_level |
| R10:boundary | informal | 4 |

3
test/bdd/steps/nominatim_environment.py
View File

@ -36,6 +36,7 @@ class NominatimEnvironment:
self.api_test_db = config['API_TEST_DB']
self.api_test_file = config['API_TEST_FILE']
self.tokenizer = config['TOKENIZER']
self.import_style = config['STYLE']
self.server_module_path = config['SERVER_MODULE_PATH']
self.reuse_template = not config['REMOVE_TEMPLATE']
self.keep_scenario_db = config['KEEP_TEST_DB']
@ -107,6 +108,8 @@ class NominatimEnvironment:
self.test_env['NOMINATIM_NOMINATIM_TOOL'] = str((self.build_dir / 'nominatim').resolve())
if self.tokenizer is not None:
self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer
if self.import_style is not None:
self.test_env['NOMINATIM_IMPORT_STYLE'] = self.import_style
if self.server_module_path:
self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path

6
test/bdd/steps/place_inserter.py
View File

@ -92,6 +92,12 @@ class PlaceColumn:
else:
self.columns[column] = {key: value}
def db_delete(self, cursor):
""" Issue a delete for the given OSM object.
"""
cursor.execute('DELETE FROM place WHERE osm_type = %s and osm_id = %s',
(self.columns['osm_type'] , self.columns['osm_id']))
def db_insert(self, cursor):
""" Insert the collected data into the database.
"""

57
test/bdd/steps/steps_db_ops.py
View File

@ -118,7 +118,10 @@ def update_place_table(context):
context.nominatim.run_nominatim('refresh', '--functions')
with context.db.cursor() as cur:
for row in context.table:
PlaceColumn(context).add_row(row, False).db_insert(cur)
col = PlaceColumn(context).add_row(row, False)
col.db_delete(cur)
col.db_insert(cur)
cur.execute('SELECT flush_deleted_places()')
context.nominatim.reindex_placex(context.db)
check_database_integrity(context)
@ -143,8 +146,10 @@ def delete_places(context, oids):
"""
context.nominatim.run_nominatim('refresh', '--functions')
with context.db.cursor() as cur:
cur.execute('TRUNCATE place_to_be_deleted')
for oid in oids.split(','):
NominatimID(oid).query_osm_id(cur, 'DELETE FROM place WHERE {}')
cur.execute('SELECT flush_deleted_places()')
context.nominatim.reindex_placex(context.db)
@ -185,7 +190,10 @@ def check_place_contents(context, table, exact):
if exact:
cur.execute('SELECT osm_type, osm_id, class from {}'.format(table))
assert expected_content == set([(r[0], r[1], r[2]) for r in cur])
actual = set([(r[0], r[1], r[2]) for r in cur])
assert expected_content == actual, \
f"Missing entries: {expected_content - actual}\n" \
f"Not expected in table: {actual - expected_content}"
@then("(?P<table>placex|place) has no entry for (?P<oid>.*)")
@ -372,4 +380,49 @@ def check_location_property_osmline(context, oid, neg):
assert not todo, f"Unmatched lines in table: {list(context.table[i] for i in todo)}"
@then("location_property_osmline contains(?P<exact> exactly)?")
def check_place_contents(context, exact):
""" Check contents of the interpolation table. Each row represents a table row
and all data must match. Data not present in the expected table, may
be arbitry. The rows are identified via the 'object' column which must
have an identifier of the form '<osm id>[:<startnumber>]'. When multiple
rows match (for example because 'startnumber' was left out and there are
multiple entries for the given OSM object) then all must match. All
expected rows are expected to be present with at least one database row.
When 'exactly' is given, there must not be additional rows in the database.
"""
with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
expected_content = set()
for row in context.table:
if ':' in row['object']:
nid, start = row['object'].split(':', 2)
start = int(start)
else:
nid, start = row['object'], None
query = """SELECT *, ST_AsText(linegeo) as geomtxt,
ST_GeometryType(linegeo) as geometrytype
FROM location_property_osmline WHERE osm_id=%s"""
if ':' in row['object']:
query += ' and startnumber = %s'
params = [int(val) for val in row['object'].split(':', 2)]
else:
params = (int(row['object']), )
cur.execute(query, params)
assert cur.rowcount > 0, "No rows found for " + row['object']
for res in cur:
if exact:
expected_content.add((res['osm_id'], res['startnumber']))
DBRow(nid, res, context).assert_row(row, ['object'])
if exact:
cur.execute('SELECT osm_id, startnumber from location_property_osmline')
actual = set([(r[0], r[1]) for r in cur])
assert expected_content == actual, \
f"Missing entries: {expected_content - actual}\n" \
f"Not expected in table: {actual - expected_content}"

15
test/bdd/steps/steps_osm_data.py
View File

@ -10,6 +10,7 @@ import os
from pathlib import Path
from nominatim.tools.exec_utils import run_osm2pgsql
from nominatim.tools.replication import run_osm2pgsql_updates
from geometry_alias import ALIASES
@ -17,7 +18,8 @@ def get_osm2pgsql_options(nominatim_env, fname, append):
return dict(import_file=fname,
osm2pgsql=str(nominatim_env.build_dir / 'osm2pgsql' / 'osm2pgsql'),
osm2pgsql_cache=50,
osm2pgsql_style=str(nominatim_env.src_dir / 'settings' / 'import-extratags.style'),
osm2pgsql_style=str(nominatim_env.get_test_config().get_import_style_file()),
osm2pgsql_style_path=nominatim_env.get_test_config().config_dir,
threads=1,
dsn=nominatim_env.get_libpq_dsn(),
flatnode_file='',
@ -117,6 +119,15 @@ def update_from_osm_file(context):
# create an OSM file and import it
fname = write_opl_file(context.text, context.osm)
try:
run_osm2pgsql(get_osm2pgsql_options(context.nominatim, fname, append=True))
run_osm2pgsql_updates(context.db,
get_osm2pgsql_options(context.nominatim, fname, append=True))
finally:
os.remove(fname)
@when('indexing')
def index_database(context):
"""
Run the Nominatim indexing step. This will process data previously
loaded with 'updating osm data'
"""
context.nominatim.run_nominatim('index')

2
vagrant/Install-on-Ubuntu-18.sh
View File

@ -24,7 +24,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev\
libbz2-dev libpq-dev \
libbz2-dev libpq-dev liblua5.3-dev lua5.3\
postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
php-cli php-pgsql php-intl libicu-dev python3-pip \

2
vagrant/Install-on-Ubuntu-20.sh
View File

@ -23,7 +23,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
libbz2-dev libpq-dev \
libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
postgresql-12-postgis-3 \
postgresql-contrib-12 postgresql-12-postgis-3-scripts \
php-cli php-pgsql php-intl libicu-dev python3-dotenv \

2
vagrant/Install-on-Ubuntu-22.sh
View File

@ -23,7 +23,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
sudo apt install -y php-cgi
sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
libbz2-dev libpq-dev \
libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
postgresql-server-dev-14 postgresql-14-postgis-3 \
postgresql-contrib-14 postgresql-14-postgis-3-scripts \
php-cli php-pgsql php-intl libicu-dev python3-dotenv \