Merge pull request #3861 from lonvia/force-extra-tags

Force inclusion of extra tags when Nominatim internally depends on them

docs/admin/Migration.md

@@ -17,6 +17,18 @@ breaking changes. **Please read them before running the migration.**
     and migrate to 4.3 first. Then you can migrate to the current
     version. It is strongly recommended to do a reimport instead.
 
+## 5.1.0 -> 5.2.0
+
+### Lua import style: required extratags removed
+
+Tags that are required by Nominatim as extratags are now always included
+independent of what is defined in the style. The line
+
+    flex.add_for_extratags('required')
+
+is no longer required in custom styles and will throw an error. Simply
+remove the line from your style.
+
 ## 4.5.0 -> 5.0.0
 
 ### PHP frontend removed

docs/customize/Import-Styles.md

@@ -76,8 +76,8 @@ and may be returned with the result [on request](../api/Search.md#output-details
 
 !!! danger
     Some tags in the extratags category are used by Nominatim to better
-    classify the place. You want to make sure these are always present
-    in custom styles.
+    classify the place. These tags will always be added, independent of
+    any settings in the style.
 
 Configuring the style means deciding which key and/or key/value is used
 in which category.
@@ -267,11 +267,7 @@ in turn take precedence over prefix matches.
 
 ##### Presets
 
-| Name     | Description |
-| :-----   | :---------- |
-| required | Tags that Nominatim will use for various computations when present in extratags. Always include these. |
-
-In addition, all [presets from ignored tags](#presets_1) are accepted.
+Accepts all [presets from ignored tags](#presets_1).
 
 ### General pre-filtering
 
@@ -462,9 +458,9 @@ with a table with the following fields:
   will always be excluded, independently of this setting.
 
 To have even more fine-grained control over the output, you can also hand
-in a table with a single field `func` containing a callback for processing
-entrance information. The callback function receives a single parameter,
-the [osm2pgsql object](https://osm2pgsql.org/doc/manual.html#processing-callbacks).
+in a callback for processing entrance information. The callback function
+receives a single parameter, the
+[osm2pgsql object](https://osm2pgsql.org/doc/manual.html#processing-callbacks).
 This object itself must not be modified. The callback should return either
 `nil` when the object is not an entrance. Or it returns a table with a
 mandatory `entrance` field containing a string with the type of entrance

lib-lua/themes/nominatim/init.lua

@@ -29,6 +29,7 @@ local NAME_FILTER = nil
 local ADDRESS_TAGS = {}
 local ADDRESS_FILTER = nil
 local EXTRATAGS_FILTER
+local REQUIRED_EXTRATAGS_FILTER
 local POSTCODE_FALLBACK = true
 local ENTRANCE_FUNCTION = nil
 
@@ -165,24 +166,6 @@ local function address_fallback(place)
     return place:clone{names=names}
 end
 
---------- Built-in extratags transformation functions ---------------
-
-local function default_extratags_filter(p, k)
-    -- Default handling is to copy over place tag for boundaries.
-    -- Nominatim needs this.
-    if k ~= 'boundary' or p.intags.place == nil then
-        return p.extratags
-    end
-
-    local extra = { place = p.intags.place }
-    for kin, vin in pairs(p.extratags) do
-        extra[kin] = vin
-    end
-
-    return extra
-end
-EXTRATAGS_FILTER = default_extratags_filter
-
 ----------------- other helper functions -----------------------------
 
 local function lookup_prefilter_classification(k, v)
@@ -444,10 +427,17 @@ function Place:write_row(k, v)
         return 0
     end
 
-    local extratags = EXTRATAGS_FILTER(self, k, v)
-    if not (extratags and next(extratags)) then
-        extratags = nil
-    end
+     local extra = EXTRATAGS_FILTER(self, k, v) or {}
+
+     for tk, tv in pairs(self.object.tags) do
+         if REQUIRED_EXTRATAGS_FILTER(tk, tv) and extra[tk] == nil then
+             extra[tk] = tv
+         end
+     end
+
+     if extra and next(extra) == nil then
+         extra = nil
+     end
 
     insert_row.place{
         class = k,
@@ -455,7 +445,7 @@ function Place:write_row(k, v)
         admin_level = self.admin_level,
         name = next(self.names) and self.names,
         address = next(self.address) and self.address,
-        extratags = extratags,
+        extratags = extra,
         geometry = self.geometry
     }
 
@@ -712,6 +702,15 @@ function module.process_tags(o)
     end
 end
 
+--------- Extratags post-processing functions ---------------
+
+local function default_extratags_filter(p, k)
+    return p.extratags
+end
+
+EXTRATAGS_FILTER = default_extratags_filter
+REQUIRED_EXTRATAGS_FILTER = module.tag_match(PRESETS.EXTRATAGS)
+
 --------- Convenience functions for simple style configuration -----------------
 
 function module.set_prefilters(data)
@@ -742,7 +741,7 @@ end
 function module.add_for_extratags(data)
     if type(data) == 'string' then
         local preset = data
-        data = PRESETS.EXTRATAGS[data] or PRESETS.IGNORE_KEYS[data]
+        data = PRESETS.IGNORE_KEYS[data]
         if data == nil then
             error('Unknown preset for extratags: ' .. preset)
         end
@@ -943,17 +942,22 @@ function module.set_relation_types(data)
 end
 
 function module.set_entrance_filter(data)
+    if data == nil or type(data) == 'function' then
+        ENTRANCE_FUNCTION = data
+        return nil
+    end
+
     if type(data) == 'string' then
         local preset = data
-        data = PRESETS.ENTRACE_TABLE[data]
+        data = PRESETS.ENTRANCE_TABLE[data]
         if data == nil then
             error('Unknown preset for entrance table: ' .. preset)
         end
     end
 
-    ENTRANCE_FUNCTION = data and data.func
+    ENTRANCE_FUNCTION = nil
 
-    if data ~= nil and data.main_tags ~= nil and next(data.main_tags) ~= nil then
+    if data.main_tags ~= nil and next(data.main_tags) ~= nil then
         if data.extra_include ~= nil and next(data.extra_include) == nil then
             -- shortcut: no extra tags requested
             ENTRANCE_FUNCTION = function(o)

lib-lua/themes/nominatim/presets.lua

@@ -375,17 +375,15 @@ module.IGNORE_KEYS.address = {'addr:street:*', 'addr:city:*', 'addr:district:*',
                               'addr:province:*', 'addr:subdistrict:*', 'addr:place:*',
                               'addr:TW:dataset'}
 
--- Extra tags (prefiltered away)
+-- INTERNAL: Required extra tags
 
-module.EXTRATAGS = {}
-
-module.EXTRATAGS.required = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
+module.EXTRATAGS = {keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}}
 
 -- Defaults for the entrance table
 
-module.ENTRACE_TABLE = {}
+module.ENTRANCE_TABLE = {}
 
-module.ENTRACE_TABLE.default = {main_tags = {'entrance', 'routing:entrance'},
-                                extra_exclude = module.IGNORE_KEYS.metatags}
+module.ENTRANCE_TABLE.default = {main_tags = {'entrance', 'routing:entrance'},
+                                 extra_exclude = module.IGNORE_KEYS.metatags}
 
 return module

lib-lua/themes/nominatim/topics/address.lua

@@ -11,7 +11,6 @@ flex.set_address_tags('core')
 flex.modify_address_tags('houses')
 
 flex.ignore_keys('metatags')
-flex.add_for_extratags('required')
 
 if cfg.with_extratags then
     flex.set_unused_handling{delete_keys = {'tiger:*'}}

lib-lua/themes/nominatim/topics/admin.lua

@@ -8,7 +8,6 @@ flex.set_address_tags('core')
 flex.set_postcode_fallback(false)
 
 flex.ignore_keys('metatags')
-flex.add_for_extratags('required')
 
 if cfg.with_extratags then
     flex.set_unused_handling{delete_keys = {'tiger:*'}}

lib-lua/themes/nominatim/topics/full.lua

@@ -20,7 +20,6 @@ flex.set_address_tags('core')
 flex.modify_address_tags('houses')
 
 flex.ignore_keys('metatags')
-flex.add_for_extratags('required')
 
 if cfg.with_extratags then
     flex.set_unused_handling{delete_keys = {'tiger:*'}}

lib-lua/themes/nominatim/topics/street.lua

@@ -10,7 +10,6 @@ flex.set_address_tags('core')
 flex.set_postcode_fallback(false)
 
 flex.ignore_keys('metatags')
-flex.add_for_extratags('required')
 
 if cfg.with_extratags then
     flex.set_unused_handling{delete_keys = {'tiger:*'}}

test/bdd/features/osm2pgsql/import/entrances.feature

@@ -109,9 +109,9 @@ Feature: Import of entrance objects by osm2pgsql
         Given the lua style file
             """
             local flex = require('import-full')
-            flex.set_entrance_filter{func = function(object)
+            flex.set_entrance_filter(function(object)
                 return {entrance='always', extratags = {ref = '1'}}
-            end}
+            end)
             """
         When loading osm data
           """

test/bdd/features/osm2pgsql/import/simple.feature

@@ -40,3 +40,26 @@ Feature: Import of simple objects by osm2pgsql
         Then place contains exactly
           | object | class | type |
           | N1     | place | house |
+
+    Scenario Outline: Tags used by Nominatim internally are always imported
+        Given the lua style file
+            """
+            local flex = require('import-<style>')
+            """
+        When loading osm data
+            """
+            n1 Tboundary=administrative,place=city,name=Foo,wikipedia:de=Foo
+            n2 Tplace=hamlet,wikidata=Q1234321,name=Bar
+            """
+        Then place contains exactly
+           | object | class    | extratags!dict         |
+           | N1     | boundary | 'place': 'city', 'wikipedia:de': 'Foo' |
+           | N2     | place    | 'wikidata': 'Q1234321' |
+
+        Examples:
+           | style   |
+           | admin   |
+           | street  |
+           | address |
+           | full    |
+           | extratags |