force inclusion of extratags used directly by Nominatim

docs/admin/Migration.md

@@ -17,6 +17,18 @@ breaking changes. **Please read them before running the migration.**
     and migrate to 4.3 first. Then you can migrate to the current
     version. It is strongly recommended to do a reimport instead.
 
+## 5.1.0 -> 5.2.0
+
+### Lua import style: required extratags removed
+
+Tags that are required by Nominatim as extratags are now always included
+independent of what is defined in the style. The line
+
+    flex.add_for_extratags('required')
+
+is no longer required in custom styles and will throw an error. Simply
+remove the line from your style.
+
 ## 4.5.0 -> 5.0.0
 
 ### PHP frontend removed

docs/customize/Import-Styles.md

@@ -267,11 +267,7 @@ in turn take precedence over prefix matches.
 
 ##### Presets
 
-| Name     | Description |
+Accepts all [presets from ignored tags](#presets_1).
-| :-----   | :---------- |
-| required | Tags that Nominatim will use for various computations when present in extratags. Automatically added. |
-
-In addition, all [presets from ignored tags](#presets_1) are accepted.
 
 ### General pre-filtering
 

lib-lua/themes/nominatim/init.lua

@@ -29,6 +29,7 @@ local NAME_FILTER = nil
 local ADDRESS_TAGS = {}
 local ADDRESS_FILTER = nil
 local EXTRATAGS_FILTER
+local REQUIRED_EXTRATAGS_FILTER
 local POSTCODE_FALLBACK = true
 local ENTRANCE_FUNCTION = nil
 
@@ -165,24 +166,6 @@ local function address_fallback(place)
     return place:clone{names=names}
 end
 
---------- Built-in extratags transformation functions ---------------
-
-local function default_extratags_filter(p, k)
-    -- Default handling is to copy over place tag for boundaries.
-    -- Nominatim needs this.
-    if k ~= 'boundary' or p.intags.place == nil then
-        return p.extratags
-    end
-
-    local extra = { place = p.intags.place }
-    for kin, vin in pairs(p.extratags) do
-        extra[kin] = vin
-    end
-
-    return extra
-end
-EXTRATAGS_FILTER = default_extratags_filter
-
 ----------------- other helper functions -----------------------------
 
 local function lookup_prefilter_classification(k, v)
@@ -444,10 +427,17 @@ function Place:write_row(k, v)
         return 0
     end
 
-    local extratags = EXTRATAGS_FILTER(self, k, v)
+     local extra = EXTRATAGS_FILTER(self, k, v) or {}
-    if not (extratags and next(extratags)) then
+
-        extratags = nil
+     for tk, tv in pairs(self.object.tags) do
-    end
+         if REQUIRED_EXTRATAGS_FILTER(tk, tv) and extra[tk] == nil then
+             extra[tk] = tv
+         end
+     end
+
+     if extra and next(extra) == nil then
+         extra = nil
+     end
 
     insert_row.place{
         class = k,
@@ -455,7 +445,7 @@ function Place:write_row(k, v)
         admin_level = self.admin_level,
         name = next(self.names) and self.names,
         address = next(self.address) and self.address,
-        extratags = extratags,
+        extratags = extra,
         geometry = self.geometry
     }
 
@@ -712,6 +702,15 @@ function module.process_tags(o)
     end
 end
 
+--------- Extratags post-processing functions ---------------
+
+local function default_extratags_filter(p, k)
+    return p.extratags
+end
+
+EXTRATAGS_FILTER = default_extratags_filter
+REQUIRED_EXTRATAGS_FILTER = module.tag_match(PRESETS.EXTRATAGS)
+
 --------- Convenience functions for simple style configuration -----------------
 
 function module.set_prefilters(data)
@@ -742,7 +741,7 @@ end
 function module.add_for_extratags(data)
     if type(data) == 'string' then
         local preset = data
-        data = PRESETS.EXTRATAGS[data] or PRESETS.IGNORE_KEYS[data]
+        data = PRESETS.IGNORE_KEYS[data]
         if data == nil then
             error('Unknown preset for extratags: ' .. preset)
         end

lib-lua/themes/nominatim/presets.lua

@@ -375,11 +375,9 @@ module.IGNORE_KEYS.address = {'addr:street:*', 'addr:city:*', 'addr:district:*',
                               'addr:province:*', 'addr:subdistrict:*', 'addr:place:*',
                               'addr:TW:dataset'}
 
--- Extra tags (prefiltered away)
+-- INTERNAL: Required extra tags
 
-module.EXTRATAGS = {}
+module.EXTRATAGS = {keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}}
-
-module.EXTRATAGS.required = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
 
 -- Defaults for the entrance table
 

lib-lua/themes/nominatim/topics/address.lua

@@ -11,7 +11,6 @@ flex.set_address_tags('core')
 flex.modify_address_tags('houses')
 
 flex.ignore_keys('metatags')
-flex.add_for_extratags('required')
 
 if cfg.with_extratags then
     flex.set_unused_handling{delete_keys = {'tiger:*'}}

lib-lua/themes/nominatim/topics/admin.lua

@@ -8,7 +8,6 @@ flex.set_address_tags('core')
 flex.set_postcode_fallback(false)
 
 flex.ignore_keys('metatags')
-flex.add_for_extratags('required')
 
 if cfg.with_extratags then
     flex.set_unused_handling{delete_keys = {'tiger:*'}}

lib-lua/themes/nominatim/topics/full.lua

@@ -20,7 +20,6 @@ flex.set_address_tags('core')
 flex.modify_address_tags('houses')
 
 flex.ignore_keys('metatags')
-flex.add_for_extratags('required')
 
 if cfg.with_extratags then
     flex.set_unused_handling{delete_keys = {'tiger:*'}}

lib-lua/themes/nominatim/topics/street.lua

@@ -10,7 +10,6 @@ flex.set_address_tags('core')
 flex.set_postcode_fallback(false)
 
 flex.ignore_keys('metatags')
-flex.add_for_extratags('required')
 
 if cfg.with_extratags then
     flex.set_unused_handling{delete_keys = {'tiger:*'}}

test/bdd/features/osm2pgsql/import/simple.feature

@@ -40,3 +40,26 @@ Feature: Import of simple objects by osm2pgsql
         Then place contains exactly
           | object | class | type |
           | N1     | place | house |
+
+    Scenario Outline: Tags used by Nominatim internally are always imported
+        Given the lua style file
+            """
+            local flex = require('import-<style>')
+            """
+        When loading osm data
+            """
+            n1 Tboundary=administrative,place=city,name=Foo,wikipedia:de=Foo
+            n2 Tplace=hamlet,wikidata=Q1234321,name=Bar
+            """
+        Then place contains exactly
+           | object | class    | extratags!dict         |
+           | N1     | boundary | 'place': 'city', 'wikipedia:de': 'Foo' |
+           | N2     | place    | 'wikidata': 'Q1234321' |
+
+        Examples:
+           | style   |
+           | admin   |
+           | street  |
+           | address |
+           | full    |
+           | extratags |