add tests for examples in lua style documentation

And fix all the errors the tests have found.

docs/customize/Import-Styles.md

@@ -49,7 +49,9 @@ during the processing.
     There are no built-in defaults for the tag lists, so all the functions
     need to be called from your style script to fully process the data.
     Make sure you start from one of the default style and only modify
-    the data you are interested in.
+    the data you are interested in. You can also derive your style from an
+    existing style by importing the appropriate module, e.g.
+    `local flex = require('import-street')`.
 
 Many of the following functions take _key match lists_. These lists can
 contain three kinds of strings to match against tag keys:
@@ -91,10 +93,12 @@ key, then this is used as default for values that are not listed.
 
 !!! example
     ``` lua
+    local flex = require('import-full')
+
     flex.set_main_tags{
-        boundary = {'administrative' = named},
+        boundary = {administrative = 'named'},
-        highway = {'always', street_lamp = 'named'}
+        highway = {'always', street_lamp = 'named'},
-        landuse = 'fallback',
+        landuse = 'fallback'
     }
     ```
 
@@ -134,9 +138,11 @@ Any other string is matched exactly against tag keys.
 
 !!! example
     ``` lua
+    local flex = require('import-full')
+
     flex.set_prefilters{
         delete_keys = {'source', 'source:*'},
-        extra_tags = {'amenity' = {'yes', 'no'}
+        extra_tags = {amenity = {'yes', 'no'}}
     }
     flex.set_main_tags{
         amenity = 'always'
@@ -168,7 +174,9 @@ They take _key match lists_ for main and extra names respectively.
 
 !!! example
     ``` lua
-    flex.set_main_tags{'highway' = {'traffic_light' = 'named'}}
+    local flex = require('flex-base')
+
+    flex.set_main_tags{highway = {traffic_light = 'named'}}
     flex.set_name_tags{main = {'name', 'name:*'},
                        extra = {'ref'}
                       }
@@ -204,6 +212,8 @@ are accepted, all other values are discarded.
 
 !!! example
     ``` lua
+    local flex = require('import-full')
+
     flex.set_address_tags{
         main = {'addr:housenumber'},
         extra = {'addr:*'},
@@ -239,6 +249,8 @@ above.
 
 !!! example
     ``` lua
+    local flex = require('import-full')
+
     flex.set_address_tags{
         main = {'addr:housenumber'},
         extra = {'addr:*', 'tiger:county'}
@@ -274,6 +286,8 @@ kinds of geometries can be used:
 
 !!! Example
     ``` lua
+    local flex = require('import-full')
+
     flex.RELATION_TYPES['site'] = flex.relation_as_multipolygon
     ```
 
@@ -292,6 +306,8 @@ logic.
 
 !!! Example
     ``` lua
+    local flex = require('import-full')
+
     function osm2pgsql.process_relation(object)
         if object.tags.boundary ~= 'administrative' or object.tags.admin_level ~= '2' then
           flex.process_relation(object)
@@ -312,6 +328,8 @@ responsible for filtering the tags and writing out the rows into Postgresql.
 
 !!! Example
     ``` lua
+    local flex = require('import-full')
+
     local original_process_tags = flex.process_tags
 
     function flex.process_tags(o)

nominatim/tools/exec_utils.py

@@ -125,7 +125,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
           ]
 
     if str(options['osm2pgsql_style']).endswith('.lua'):
-        env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / 'flex-base.lua'),
+        env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'),
                                     os.environ.get('LUAPATH', ';')))
         cmd.extend(('--output', 'flex'))
     else:

settings/flex-base.lua

@@ -461,8 +461,8 @@ end
 
 function module.set_prefilters(data)
     PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
-    PRE_EXTRAS = module.tag_match{keys = data.extratag_keys,
+    PRE_EXTRAS = module.tag_match{keys = data.extra_keys,
-                                  tags = data.extratag_tags}
+                                  tags = data.extra_tags}
 end
 
 function module.set_main_tags(data)
@@ -484,12 +484,12 @@ end
 
 function module.set_unused_handling(data)
     if data.extra_keys == nil and data.extra_tags == nil then
-        POST_DELETE = module.tag_match{data.delete_keys, tags = data.delete_tags}
+        POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
         POST_EXTRAS = nil
         SAVE_EXTRA_MAINS = true
     elseif data.delete_keys == nil and data.delete_tags == nil then
         POST_DELETE = nil
-        POST_EXTRAS = module.tag_match{data.extra_keys, tags = data.extra_tags}
+        POST_EXTRAS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags}
         SAVE_EXTRA_MAINS = false
     else
         error("unused handler can have only 'extra_keys' or 'delete_keys' set.")

settings/import-address.lua

@@ -1,4 +1,4 @@
-flex = require('flex-base')
+local flex = require('flex-base')
 
 flex.set_main_tags{
     highway = {'always',
@@ -32,7 +32,7 @@ flex.set_prefilters{delete_keys = {'building', 'source',
                                               'noexit', 'crossing', 'give_way', 'stop'},
                                    landuse = {'cemetry', 'no'},
                                    boundary = {'place'}},
-                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
+                    extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
                    }
 
 flex.set_name_tags{main = {'name', 'name:*',
@@ -65,3 +65,5 @@ flex.set_address_tags{main = {'addr:housenumber',
 
 
 flex.set_unused_handling{extra_keys = {'place'}}
+
+return flex

settings/import-admin.lua

@@ -15,7 +15,7 @@ flex.set_prefilters{delete_keys = {'building', 'source', 'highway',
                                    'addr:street:name', 'addr:street:type'},
                     delete_tags = {landuse = {'cemetry', 'no'},
                                    boundary = {'place'}},
-                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
+                    extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
                    }
 
 flex.set_name_tags{main = {'name', 'name:*',
@@ -42,3 +42,5 @@ flex.set_address_tags{extra = {'addr:*', 'is_in:*'},
                      }
 
 flex.set_unused_handling{extra_keys = {'place'}}
+
+return flex

settings/import-extratags.lua

@@ -1,4 +1,4 @@
-flex = require('flex-base')
+local flex = require('flex-base')
 
 flex.set_main_tags{
     building = 'fallback',
@@ -74,7 +74,7 @@ flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attri
                                    waterway = {'riverbank'},
                                    building = {'no'},
                                    boundary = {'place'}},
-                    extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                    extra_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
                                'name:etymology', 'name:signed', 'name:botanical',
                                'wikidata', '*:wikidata',
                                'addr:street:name', 'addr:street:type'}
@@ -110,3 +110,5 @@ flex.set_address_tags{main = {'addr:housenumber',
 
 
 flex.set_unused_handling{delete_keys = {'tiger:*'}}
+
+return flex

settings/import-full.lua

@@ -1,4 +1,4 @@
-flex = require('flex-base')
+local flex = require('flex-base')
 
 flex.set_main_tags{
     building = 'fallback',
@@ -74,7 +74,7 @@ flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attri
                                    waterway = {'riverbank'},
                                    building = {'no'},
                                    boundary = {'place'}},
-                    extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                    extra_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
                                'name:etymology', 'name:signed', 'name:botanical',
                                'wikidata', '*:wikidata',
                                'addr:street:name', 'addr:street:type'}
@@ -110,3 +110,5 @@ flex.set_address_tags{main = {'addr:housenumber',
 
 
 flex.set_unused_handling{extra_keys = {'place'}}
+
+return flex

settings/import-street.lua

@@ -1,4 +1,4 @@
-flex = require('flex-base')
+local flex = require('flex-base')
 
 flex.set_main_tags{
     highway = {'always',
@@ -32,7 +32,7 @@ flex.set_prefilters{delete_keys = {'building', 'source',
                                               'noexit', 'crossing', 'give_way', 'stop'},
                                    landuse = {'cemetry', 'no'},
                                    boundary = {'place'}},
-                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
+                    extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
                    }
 
 flex.set_name_tags{main = {'name', 'name:*',
@@ -64,3 +64,5 @@ flex.set_address_tags{main = {'addr:housenumber',
                      }
 
 flex.set_unused_handling{extra_keys = {'place'}}
+
+return flex

test/bdd/osm2pgsql/import/custom_style.feature

@@ -0,0 +1,200 @@
+@DB
+Feature: Import with custom styles by osm2pgsql
+    Tests for the example customizations given in the documentation.
+
+    Scenario: Custom main tags
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.set_main_tags{
+                boundary = {administrative = 'named'},
+                highway = {'always', street_lamp = 'named'},
+                landuse = 'fallback'
+            }
+            """
+        When loading osm data
+            """
+            n10 Tboundary=administrative x0 y0
+            n11 Tboundary=administrative,name=Foo x0 y0
+            n12 Tboundary=electoral x0 y0
+            n13 Thighway=primary x0 y0
+            n14 Thighway=street_lamp x0 y0
+            n15 Thighway=primary,landuse=street x0 y0
+            """
+        Then place contains exactly
+            | object | class    | type           |
+            | N11    | boundary | administrative |
+            | N13    | highway  | primary        |
+            | N15    | highway  | primary        |
+
+    Scenario: Prefiltering tags
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.set_prefilters{
+                delete_keys = {'source', 'source:*'},
+                extra_tags = {amenity = {'yes', 'no'}}
+            }
+            flex.set_main_tags{
+                amenity = 'always',
+                tourism = 'always'
+            }
+            """
+        When loading osm data
+            """
+            n1 Tamenity=yes x0 y6
+            n2 Tamenity=hospital,source=survey x3 y6
+            n3 Ttourism=hotel,amenity=yes x0 y0
+            n4 Ttourism=hotel,amenity=telephone x0 y0
+            """
+        Then place contains exactly
+            | object     | extratags              |
+            | N2:amenity | -                      |
+            | N3:tourism | 'amenity': 'yes'       |
+            | N4:tourism | - |
+            | N4:amenity | - |
+
+    Scenario: Name tags
+        Given the lua style file
+            """
+            local flex = require('flex-base')
+
+            flex.set_main_tags{highway = {traffic_light = 'named'}}
+            flex.set_name_tags{main = {'name', 'name:*'},
+                               extra = {'ref'}
+                              }
+            """
+        When loading osm data
+            """
+            n1 Thighway=stop,name=Something x0 y0
+            n2 Thighway=traffic_light,ref=453-4 x0 y0
+            n3 Thighway=traffic_light,name=Greens x0 y0
+            n4 Thighway=traffic_light,name=Red,ref=45 x0 y0
+            """
+        Then place contains exactly
+            | object     | name                       |
+            | N3:highway | 'name': 'Greens'           |
+            | N4:highway | 'name': 'Red', 'ref': '45' |
+
+    Scenario: Address tags
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.set_address_tags{
+                main = {'addr:housenumber'},
+                extra = {'addr:*'},
+                postcode = {'postal_code', 'postcode', 'addr:postcode'},
+                country = {'country-code', 'ISO3166-1'}
+            }
+            """
+        When loading osm data
+            """
+            n1 Ttourism=hotel,addr:street=Foo x0 y0
+            n2 Taddr:housenumber=23,addr:street=Budd,postal_code=5567 x0 y0
+            n3 Taddr:street=None,addr:city=Where x0 y0
+            """
+        Then place contains exactly
+            | object     | type  | address |
+            | N1:tourism | hotel | 'street': 'Foo' |
+            | N2:place   | house | 'housenumber': '23', 'street': 'Budd', 'postcode': '5567' |
+
+    Scenario: Unused handling
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.set_address_tags{
+                main = {'addr:housenumber'},
+                extra = {'addr:*', 'tiger:county'}
+            }
+            flex.set_unused_handling{delete_keys = {'tiger:*'}}
+            """
+        When loading osm data
+            """
+            n1 Ttourism=hotel,tiger:county=Fargo x0 y0
+            n2 Ttourism=hotel,tiger:xxd=56,else=other x0 y0
+            """
+        Then place contains exactly
+            | object     | type  | address                 | extratags        |
+            | N1:tourism | hotel | 'tiger:county': 'Fargo' | -                |
+            | N2:tourism | hotel | -                       | 'else': 'other'  |
+
+    Scenario: Additional relation types
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            flex.RELATION_TYPES['site'] = flex.relation_as_multipolygon
+            """
+        And the grid
+            | 1 | 2 |
+            | 4 | 3 |
+        When loading osm data
+            """
+            n1
+            n2
+            n3
+            n4
+            w1 Nn1,n2,n3,n4,n1
+            r1 Ttype=multipolygon,amenity=school Mw1@
+            r2 Ttype=site,amenity=school Mw1@
+            """
+        Then place contains exactly
+            | object     | type   |
+            | R1:amenity | school |
+            | R2:amenity | school |
+
+    Scenario: Exclude country relations
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            function osm2pgsql.process_relation(object)
+                if object.tags.boundary ~= 'administrative' or object.tags.admin_level ~= '2' then
+                  flex.process_relation(object)
+                end
+            end
+            """
+       And the grid
+            | 1 | 2 |
+            | 4 | 3 |
+       When loading osm data
+            """
+            n1
+            n2
+            n3
+            n4
+            w1 Nn1,n2,n3,n4,n1
+            r1 Ttype=multipolygon,boundary=administrative,admin_level=4,name=Small Mw1@
+            r2 Ttype=multipolygon,boundary=administrative,admin_level=2,name=Big Mw1@
+            """
+        Then place contains exactly
+            | object      | type           |
+            | R1:boundary | administrative |
+
+    Scenario: Customize processing functions
+        Given the lua style file
+            """
+            local flex = require('import-full')
+
+            local original_process_tags = flex.process_tags
+
+            function flex.process_tags(o)
+                if o.object.tags.highway ~= nil and o.object.tags.access == 'no' then
+                    return
+                end
+
+                original_process_tags(o)
+            end
+            """
+        When loading osm data
+            """
+            n1 Thighway=residential x0 y0
+            n2 Thighway=residential,access=no x0 y0
+            """
+        Then place contains exactly
+            | object     | type        |
+            | N1:highway | residential |

test/bdd/steps/steps_osm_data.py

@@ -49,6 +49,15 @@ def write_opl_file(opl, grid):
 
         return fd.name
 
+@given('the lua style file')
+def lua_style_file(context):
+    """ Define a custom style file to use for the import.
+    """
+    style = Path(context.nominatim.website_dir.name) / 'custom.lua'
+    style.write_text(context.text)
+    context.nominatim.test_env['NOMINATIM_IMPORT_STYLE'] = str(style)
+
+
 @given(u'the ([0-9.]+ )?grid(?: with origin (?P<origin>.*))?')
 def define_node_grid(context, grid_step, origin):
     """