osm-search/Nominatim
1
1
Fork 0
mirror of https://github.com/osm-search/Nominatim.git synced 2024-11-22 12:06:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

add tests for examples in lua style documentation

Browse Source 
And fix all the errors the tests have found.
This commit is contained in:
Sarah Hoffmann 2022-12-23 17:35:28 +01:00
parent 9321e425a4
commit 200eae3bc0
10 changed files with 257 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
docs/customize/Import-Styles.md
View File

@ -49,7 +49,9 @@ during the processing.
There are no built-in defaults for the tag lists, so all the functions
need to be called from your style script to fully process the data.
Make sure you start from one of the default style and only modify
the data you are interested in.
the data you are interested in. You can also derive your style from an
existing style by importing the appropriate module, e.g.
`local flex = require('import-street')`.
Many of the following functions take _key match lists_. These lists can
contain three kinds of strings to match against tag keys:
@ -91,10 +93,12 @@ key, then this is used as default for values that are not listed.
!!! example
``` lua
local flex = require('import-full')
flex.set_main_tags{
boundary = {'administrative' = named},
highway = {'always', street_lamp = 'named'}
landuse = 'fallback',
boundary = {administrative = 'named'},
highway = {'always', street_lamp = 'named'},
landuse = 'fallback'
}
```
@ -134,9 +138,11 @@ Any other string is matched exactly against tag keys.
!!! example
``` lua
local flex = require('import-full')
flex.set_prefilters{
delete_keys = {'source', 'source:*'},
extra_tags = {'amenity' = {'yes', 'no'}
extra_tags = {amenity = {'yes', 'no'}}
}
flex.set_main_tags{
amenity = 'always'
@ -168,7 +174,9 @@ They take _key match lists_ for main and extra names respectively.
!!! example
``` lua
flex.set_main_tags{'highway' = {'traffic_light' = 'named'}}
local flex = require('flex-base')
flex.set_main_tags{highway = {traffic_light = 'named'}}
flex.set_name_tags{main = {'name', 'name:*'},
extra = {'ref'}
}
@ -204,6 +212,8 @@ are accepted, all other values are discarded.
!!! example
``` lua
local flex = require('import-full')
flex.set_address_tags{
main = {'addr:housenumber'},
extra = {'addr:*'},
@ -239,6 +249,8 @@ above.
!!! example
``` lua
local flex = require('import-full')
flex.set_address_tags{
main = {'addr:housenumber'},
extra = {'addr:*', 'tiger:county'}
@ -274,6 +286,8 @@ kinds of geometries can be used:
!!! Example
``` lua
local flex = require('import-full')
flex.RELATION_TYPES['site'] = flex.relation_as_multipolygon
```
@ -292,6 +306,8 @@ logic.
!!! Example
``` lua
local flex = require('import-full')
function osm2pgsql.process_relation(object)
if object.tags.boundary ~= 'administrative' or object.tags.admin_level ~= '2' then
flex.process_relation(object)
@ -312,6 +328,8 @@ responsible for filtering the tags and writing out the rows into Postgresql.
!!! Example
``` lua
local flex = require('import-full')
local original_process_tags = flex.process_tags
function flex.process_tags(o)

2
nominatim/tools/exec_utils.py
View File

@ -125,7 +125,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
]
if str(options['osm2pgsql_style']).endswith('.lua'):
env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / 'flex-base.lua'),
env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'),
os.environ.get('LUAPATH', ';')))
cmd.extend(('--output', 'flex'))
else:

8
settings/flex-base.lua
View File

@ -461,8 +461,8 @@ end
function module.set_prefilters(data)
PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
PRE_EXTRAS = module.tag_match{keys = data.extratag_keys,
tags = data.extratag_tags}
PRE_EXTRAS = module.tag_match{keys = data.extra_keys,
tags = data.extra_tags}
end
function module.set_main_tags(data)
@ -484,12 +484,12 @@ end
function module.set_unused_handling(data)
if data.extra_keys == nil and data.extra_tags == nil then
POST_DELETE = module.tag_match{data.delete_keys, tags = data.delete_tags}
POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
POST_EXTRAS = nil
SAVE_EXTRA_MAINS = true
elseif data.delete_keys == nil and data.delete_tags == nil then
POST_DELETE = nil
POST_EXTRAS = module.tag_match{data.extra_keys, tags = data.extra_tags}
POST_EXTRAS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags}
SAVE_EXTRA_MAINS = false
else
error("unused handler can have only 'extra_keys' or 'delete_keys' set.")

6
settings/import-address.lua
View File

@ -1,4 +1,4 @@
flex = require('flex-base')
local flex = require('flex-base')
flex.set_main_tags{
highway = {'always',
@ -32,7 +32,7 @@ flex.set_prefilters{delete_keys = {'building', 'source',
'noexit', 'crossing', 'give_way', 'stop'},
landuse = {'cemetry', 'no'},
boundary = {'place'}},
extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
}
flex.set_name_tags{main = {'name', 'name:*',
@ -65,3 +65,5 @@ flex.set_address_tags{main = {'addr:housenumber',
flex.set_unused_handling{extra_keys = {'place'}}
return flex

4
settings/import-admin.lua
View File

@ -15,7 +15,7 @@ flex.set_prefilters{delete_keys = {'building', 'source', 'highway',
'addr:street:name', 'addr:street:type'},
delete_tags = {landuse = {'cemetry', 'no'},
boundary = {'place'}},
extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
}
flex.set_name_tags{main = {'name', 'name:*',
@ -42,3 +42,5 @@ flex.set_address_tags{extra = {'addr:*', 'is_in:*'},
}
flex.set_unused_handling{extra_keys = {'place'}}
return flex

6
settings/import-extratags.lua
View File

@ -1,4 +1,4 @@
flex = require('flex-base')
local flex = require('flex-base')
flex.set_main_tags{
building = 'fallback',
@ -74,7 +74,7 @@ flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attri
waterway = {'riverbank'},
building = {'no'},
boundary = {'place'}},
extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
extra_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
'name:etymology', 'name:signed', 'name:botanical',
'wikidata', '*:wikidata',
'addr:street:name', 'addr:street:type'}
@ -110,3 +110,5 @@ flex.set_address_tags{main = {'addr:housenumber',
flex.set_unused_handling{delete_keys = {'tiger:*'}}
return flex

6
settings/import-full.lua
View File

@ -1,4 +1,4 @@
flex = require('flex-base')
local flex = require('flex-base')
flex.set_main_tags{
building = 'fallback',
@ -74,7 +74,7 @@ flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attri
waterway = {'riverbank'},
building = {'no'},
boundary = {'place'}},
extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
extra_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
'name:etymology', 'name:signed', 'name:botanical',
'wikidata', '*:wikidata',
'addr:street:name', 'addr:street:type'}
@ -110,3 +110,5 @@ flex.set_address_tags{main = {'addr:housenumber',
flex.set_unused_handling{extra_keys = {'place'}}
return flex

6
settings/import-street.lua
View File

@ -1,4 +1,4 @@
flex = require('flex-base')
local flex = require('flex-base')
flex.set_main_tags{
highway = {'always',
@ -32,7 +32,7 @@ flex.set_prefilters{delete_keys = {'building', 'source',
'noexit', 'crossing', 'give_way', 'stop'},
landuse = {'cemetry', 'no'},
boundary = {'place'}},
extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
}
flex.set_name_tags{main = {'name', 'name:*',
@ -64,3 +64,5 @@ flex.set_address_tags{main = {'addr:housenumber',
}
flex.set_unused_handling{extra_keys = {'place'}}
return flex

200
test/bdd/osm2pgsql/import/custom_style.feature Normal file
View File

@ -0,0 +1,200 @@
@DB
Feature: Import with custom styles by osm2pgsql
Tests for the example customizations given in the documentation.
Scenario: Custom main tags
Given the lua style file
"""
local flex = require('import-full')
flex.set_main_tags{
boundary = {administrative = 'named'},
highway = {'always', street_lamp = 'named'},
landuse = 'fallback'
}
"""
When loading osm data
"""
n10 Tboundary=administrative x0 y0
n11 Tboundary=administrative,name=Foo x0 y0
n12 Tboundary=electoral x0 y0
n13 Thighway=primary x0 y0
n14 Thighway=street_lamp x0 y0
n15 Thighway=primary,landuse=street x0 y0
"""
Then place contains exactly
| object | class | type |
| N11 | boundary | administrative |
| N13 | highway | primary |
| N15 | highway | primary |
Scenario: Prefiltering tags
Given the lua style file
"""
local flex = require('import-full')
flex.set_prefilters{
delete_keys = {'source', 'source:*'},
extra_tags = {amenity = {'yes', 'no'}}
}
flex.set_main_tags{
amenity = 'always',
tourism = 'always'
}
"""
When loading osm data
"""
n1 Tamenity=yes x0 y6
n2 Tamenity=hospital,source=survey x3 y6
n3 Ttourism=hotel,amenity=yes x0 y0
n4 Ttourism=hotel,amenity=telephone x0 y0
"""
Then place contains exactly
| object | extratags |
| N2:amenity | - |
| N3:tourism | 'amenity': 'yes' |
| N4:tourism | - |
| N4:amenity | - |
Scenario: Name tags
Given the lua style file
"""
local flex = require('flex-base')
flex.set_main_tags{highway = {traffic_light = 'named'}}
flex.set_name_tags{main = {'name', 'name:*'},
extra = {'ref'}
}
"""
When loading osm data
"""
n1 Thighway=stop,name=Something x0 y0
n2 Thighway=traffic_light,ref=453-4 x0 y0
n3 Thighway=traffic_light,name=Greens x0 y0
n4 Thighway=traffic_light,name=Red,ref=45 x0 y0
"""
Then place contains exactly
| object | name |
| N3:highway | 'name': 'Greens' |
| N4:highway | 'name': 'Red', 'ref': '45' |
Scenario: Address tags
Given the lua style file
"""
local flex = require('import-full')
flex.set_address_tags{
main = {'addr:housenumber'},
extra = {'addr:*'},
postcode = {'postal_code', 'postcode', 'addr:postcode'},
country = {'country-code', 'ISO3166-1'}
}
"""
When loading osm data
"""
n1 Ttourism=hotel,addr:street=Foo x0 y0
n2 Taddr:housenumber=23,addr:street=Budd,postal_code=5567 x0 y0
n3 Taddr:street=None,addr:city=Where x0 y0
"""
Then place contains exactly
| object | type | address |
| N1:tourism | hotel | 'street': 'Foo' |
| N2:place | house | 'housenumber': '23', 'street': 'Budd', 'postcode': '5567' |
Scenario: Unused handling
Given the lua style file
"""
local flex = require('import-full')
flex.set_address_tags{
main = {'addr:housenumber'},
extra = {'addr:*', 'tiger:county'}
}
flex.set_unused_handling{delete_keys = {'tiger:*'}}
"""
When loading osm data
"""
n1 Ttourism=hotel,tiger:county=Fargo x0 y0
n2 Ttourism=hotel,tiger:xxd=56,else=other x0 y0
"""
Then place contains exactly
| object | type | address | extratags |
| N1:tourism | hotel | 'tiger:county': 'Fargo' | - |
| N2:tourism | hotel | - | 'else': 'other' |
Scenario: Additional relation types
Given the lua style file
"""
local flex = require('import-full')
flex.RELATION_TYPES['site'] = flex.relation_as_multipolygon
"""
And the grid
| 1 | 2 |
| 4 | 3 |
When loading osm data
"""
n1
n2
n3
n4
w1 Nn1,n2,n3,n4,n1
r1 Ttype=multipolygon,amenity=school Mw1@
r2 Ttype=site,amenity=school Mw1@
"""
Then place contains exactly
| object | type |
| R1:amenity | school |
| R2:amenity | school |
Scenario: Exclude country relations
Given the lua style file
"""
local flex = require('import-full')
function osm2pgsql.process_relation(object)
if object.tags.boundary ~= 'administrative' or object.tags.admin_level ~= '2' then
flex.process_relation(object)
end
end
"""
And the grid
| 1 | 2 |
| 4 | 3 |
When loading osm data
"""
n1
n2
n3
n4
w1 Nn1,n2,n3,n4,n1
r1 Ttype=multipolygon,boundary=administrative,admin_level=4,name=Small Mw1@
r2 Ttype=multipolygon,boundary=administrative,admin_level=2,name=Big Mw1@
"""
Then place contains exactly
| object | type |
| R1:boundary | administrative |
Scenario: Customize processing functions
Given the lua style file
"""
local flex = require('import-full')
local original_process_tags = flex.process_tags
function flex.process_tags(o)
if o.object.tags.highway ~= nil and o.object.tags.access == 'no' then
return
end
original_process_tags(o)
end
"""
When loading osm data
"""
n1 Thighway=residential x0 y0
n2 Thighway=residential,access=no x0 y0
"""
Then place contains exactly
| object | type |
| N1:highway | residential |

9
test/bdd/steps/steps_osm_data.py
View File

@ -49,6 +49,15 @@ def write_opl_file(opl, grid):
return fd.name
@given('the lua style file')
def lua_style_file(context):
""" Define a custom style file to use for the import.
"""
style = Path(context.nominatim.website_dir.name) / 'custom.lua'
style.write_text(context.text)
context.nominatim.test_env['NOMINATIM_IMPORT_STYLE'] = str(style)
@given(u'the ([0-9.]+ )?grid(?: with origin (?P<origin>.*))?')
def define_node_grid(context, grid_step, origin):
"""