diff --git a/docs/admin/Migration.md b/docs/admin/Migration.md index becf2f7b..2ebb8889 100644 --- a/docs/admin/Migration.md +++ b/docs/admin/Migration.md @@ -17,6 +17,18 @@ breaking changes. **Please read them before running the migration.** and migrate to 4.3 first. Then you can migrate to the current version. It is strongly recommended to do a reimport instead. +## 5.1.0 -> 5.2.0 + +### Lua import style: required extratags removed + +Tags that are required by Nominatim as extratags are now always included +independent of what is defined in the style. The line + + flex.add_for_extratags('required') + +is no longer required in custom styles and will throw an error. Simply +remove the line from your style. + ## 4.5.0 -> 5.0.0 ### PHP frontend removed diff --git a/docs/customize/Import-Styles.md b/docs/customize/Import-Styles.md index 35ff07cf..9f0fe11e 100644 --- a/docs/customize/Import-Styles.md +++ b/docs/customize/Import-Styles.md @@ -76,8 +76,8 @@ and may be returned with the result [on request](../api/Search.md#output-details !!! danger Some tags in the extratags category are used by Nominatim to better - classify the place. You want to make sure these are always present - in custom styles. + classify the place. These tags will always be added, independent of + any settings in the style. Configuring the style means deciding which key and/or key/value is used in which category. @@ -267,11 +267,7 @@ in turn take precedence over prefix matches. ##### Presets -| Name | Description | -| :----- | :---------- | -| required | Tags that Nominatim will use for various computations when present in extratags. Always include these. | - -In addition, all [presets from ignored tags](#presets_1) are accepted. +Accepts all [presets from ignored tags](#presets_1). ### General pre-filtering @@ -462,9 +458,9 @@ with a table with the following fields: will always be excluded, independently of this setting. To have even more fine-grained control over the output, you can also hand -in a table with a single field `func` containing a callback for processing -entrance information. The callback function receives a single parameter, -the [osm2pgsql object](https://osm2pgsql.org/doc/manual.html#processing-callbacks). +in a callback for processing entrance information. The callback function +receives a single parameter, the +[osm2pgsql object](https://osm2pgsql.org/doc/manual.html#processing-callbacks). This object itself must not be modified. The callback should return either `nil` when the object is not an entrance. Or it returns a table with a mandatory `entrance` field containing a string with the type of entrance diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index 228e5c55..8720a631 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -29,6 +29,7 @@ local NAME_FILTER = nil local ADDRESS_TAGS = {} local ADDRESS_FILTER = nil local EXTRATAGS_FILTER +local REQUIRED_EXTRATAGS_FILTER local POSTCODE_FALLBACK = true local ENTRANCE_FUNCTION = nil @@ -165,24 +166,6 @@ local function address_fallback(place) return place:clone{names=names} end ---------- Built-in extratags transformation functions --------------- - -local function default_extratags_filter(p, k) - -- Default handling is to copy over place tag for boundaries. - -- Nominatim needs this. - if k ~= 'boundary' or p.intags.place == nil then - return p.extratags - end - - local extra = { place = p.intags.place } - for kin, vin in pairs(p.extratags) do - extra[kin] = vin - end - - return extra -end -EXTRATAGS_FILTER = default_extratags_filter - ----------------- other helper functions ----------------------------- local function lookup_prefilter_classification(k, v) @@ -444,10 +427,17 @@ function Place:write_row(k, v) return 0 end - local extratags = EXTRATAGS_FILTER(self, k, v) - if not (extratags and next(extratags)) then - extratags = nil - end + local extra = EXTRATAGS_FILTER(self, k, v) or {} + + for tk, tv in pairs(self.object.tags) do + if REQUIRED_EXTRATAGS_FILTER(tk, tv) and extra[tk] == nil then + extra[tk] = tv + end + end + + if extra and next(extra) == nil then + extra = nil + end insert_row.place{ class = k, @@ -455,7 +445,7 @@ function Place:write_row(k, v) admin_level = self.admin_level, name = next(self.names) and self.names, address = next(self.address) and self.address, - extratags = extratags, + extratags = extra, geometry = self.geometry } @@ -712,6 +702,15 @@ function module.process_tags(o) end end +--------- Extratags post-processing functions --------------- + +local function default_extratags_filter(p, k) + return p.extratags +end + +EXTRATAGS_FILTER = default_extratags_filter +REQUIRED_EXTRATAGS_FILTER = module.tag_match(PRESETS.EXTRATAGS) + --------- Convenience functions for simple style configuration ----------------- function module.set_prefilters(data) @@ -742,7 +741,7 @@ end function module.add_for_extratags(data) if type(data) == 'string' then local preset = data - data = PRESETS.EXTRATAGS[data] or PRESETS.IGNORE_KEYS[data] + data = PRESETS.IGNORE_KEYS[data] if data == nil then error('Unknown preset for extratags: ' .. preset) end @@ -943,17 +942,22 @@ function module.set_relation_types(data) end function module.set_entrance_filter(data) + if data == nil or type(data) == 'function' then + ENTRANCE_FUNCTION = data + return nil + end + if type(data) == 'string' then local preset = data - data = PRESETS.ENTRACE_TABLE[data] + data = PRESETS.ENTRANCE_TABLE[data] if data == nil then error('Unknown preset for entrance table: ' .. preset) end end - ENTRANCE_FUNCTION = data and data.func + ENTRANCE_FUNCTION = nil - if data ~= nil and data.main_tags ~= nil and next(data.main_tags) ~= nil then + if data.main_tags ~= nil and next(data.main_tags) ~= nil then if data.extra_include ~= nil and next(data.extra_include) == nil then -- shortcut: no extra tags requested ENTRANCE_FUNCTION = function(o) diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua index 7408d4af..12c93a14 100644 --- a/lib-lua/themes/nominatim/presets.lua +++ b/lib-lua/themes/nominatim/presets.lua @@ -375,17 +375,15 @@ module.IGNORE_KEYS.address = {'addr:street:*', 'addr:city:*', 'addr:district:*', 'addr:province:*', 'addr:subdistrict:*', 'addr:place:*', 'addr:TW:dataset'} --- Extra tags (prefiltered away) +-- INTERNAL: Required extra tags -module.EXTRATAGS = {} - -module.EXTRATAGS.required = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'} +module.EXTRATAGS = {keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}} -- Defaults for the entrance table -module.ENTRACE_TABLE = {} +module.ENTRANCE_TABLE = {} -module.ENTRACE_TABLE.default = {main_tags = {'entrance', 'routing:entrance'}, - extra_exclude = module.IGNORE_KEYS.metatags} +module.ENTRANCE_TABLE.default = {main_tags = {'entrance', 'routing:entrance'}, + extra_exclude = module.IGNORE_KEYS.metatags} return module diff --git a/lib-lua/themes/nominatim/topics/address.lua b/lib-lua/themes/nominatim/topics/address.lua index 0e813673..faa04c65 100644 --- a/lib-lua/themes/nominatim/topics/address.lua +++ b/lib-lua/themes/nominatim/topics/address.lua @@ -11,7 +11,6 @@ flex.set_address_tags('core') flex.modify_address_tags('houses') flex.ignore_keys('metatags') -flex.add_for_extratags('required') if cfg.with_extratags then flex.set_unused_handling{delete_keys = {'tiger:*'}} diff --git a/lib-lua/themes/nominatim/topics/admin.lua b/lib-lua/themes/nominatim/topics/admin.lua index 47f4e248..7db8e616 100644 --- a/lib-lua/themes/nominatim/topics/admin.lua +++ b/lib-lua/themes/nominatim/topics/admin.lua @@ -8,7 +8,6 @@ flex.set_address_tags('core') flex.set_postcode_fallback(false) flex.ignore_keys('metatags') -flex.add_for_extratags('required') if cfg.with_extratags then flex.set_unused_handling{delete_keys = {'tiger:*'}} diff --git a/lib-lua/themes/nominatim/topics/full.lua b/lib-lua/themes/nominatim/topics/full.lua index c9b1b467..489fead8 100644 --- a/lib-lua/themes/nominatim/topics/full.lua +++ b/lib-lua/themes/nominatim/topics/full.lua @@ -20,7 +20,6 @@ flex.set_address_tags('core') flex.modify_address_tags('houses') flex.ignore_keys('metatags') -flex.add_for_extratags('required') if cfg.with_extratags then flex.set_unused_handling{delete_keys = {'tiger:*'}} diff --git a/lib-lua/themes/nominatim/topics/street.lua b/lib-lua/themes/nominatim/topics/street.lua index 89bed442..5c78906f 100644 --- a/lib-lua/themes/nominatim/topics/street.lua +++ b/lib-lua/themes/nominatim/topics/street.lua @@ -10,7 +10,6 @@ flex.set_address_tags('core') flex.set_postcode_fallback(false) flex.ignore_keys('metatags') -flex.add_for_extratags('required') if cfg.with_extratags then flex.set_unused_handling{delete_keys = {'tiger:*'}} diff --git a/test/bdd/features/osm2pgsql/import/entrances.feature b/test/bdd/features/osm2pgsql/import/entrances.feature index 3999743b..0d4fb16e 100644 --- a/test/bdd/features/osm2pgsql/import/entrances.feature +++ b/test/bdd/features/osm2pgsql/import/entrances.feature @@ -109,9 +109,9 @@ Feature: Import of entrance objects by osm2pgsql Given the lua style file """ local flex = require('import-full') - flex.set_entrance_filter{func = function(object) + flex.set_entrance_filter(function(object) return {entrance='always', extratags = {ref = '1'}} - end} + end) """ When loading osm data """ diff --git a/test/bdd/features/osm2pgsql/import/simple.feature b/test/bdd/features/osm2pgsql/import/simple.feature index 217c2b7c..8581fc36 100644 --- a/test/bdd/features/osm2pgsql/import/simple.feature +++ b/test/bdd/features/osm2pgsql/import/simple.feature @@ -40,3 +40,26 @@ Feature: Import of simple objects by osm2pgsql Then place contains exactly | object | class | type | | N1 | place | house | + + Scenario Outline: Tags used by Nominatim internally are always imported + Given the lua style file + """ + local flex = require('import-