Merge pull request #2881 from lonvia/more-update-tests-for-osm2pgsql

Experimental support for osm2pgsql flex output

.github/actions/build-nominatim/action.yml

@@ -9,6 +9,10 @@ inputs:
         description: 'Additional options to hand to cmake'
         required: false
         default: ''
+    lua:
+        description: 'Version of Lua to use'
+        required: false
+        default: '5.3'
 
 runs:
     using: "composite"
@@ -21,7 +25,7 @@ runs:
           shell: bash
         - name: Install prerequisites
           run: |
-            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev
+            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION}
             if [ "x$UBUNTUVER" == "x18" ]; then
                 pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 datrie
             else
@@ -31,6 +35,7 @@ runs:
           env:
             UBUNTUVER: ${{ inputs.ubuntu }}
             CMAKE_ARGS: ${{ inputs.cmake-args }}
+            LUA_VERSION: ${{ inputs.lua }}
 
         - name: Configure
           run: mkdir build && cd build && cmake $CMAKE_ARGS ../Nominatim

CMakeLists.txt

@@ -63,7 +63,6 @@ if (BUILD_IMPORTER AND BUILD_OSM2PGSQL)
     endif()
     set(BUILD_TESTS_SAVED "${BUILD_TESTS}")
     set(BUILD_TESTS off)
-    set(WITH_LUA off CACHE BOOL "")
     add_subdirectory(osm2pgsql)
     set(BUILD_TESTS ${BUILD_TESTS_SAVED})
 endif()

lib-sql/functions/place_triggers.sql

@@ -34,6 +34,11 @@ BEGIN
     RETURN null;
   END IF;
 
+  -- Remove the place from the list of places to be deleted
+  DELETE FROM place_to_be_deleted pdel
+    WHERE pdel.osm_type = NEW.osm_type and pdel.osm_id = NEW.osm_id
+          and pdel.class = NEW.class;
+
   -- Have we already done this place?
   SELECT * INTO existing
     FROM place
@@ -42,8 +47,6 @@ BEGIN
 
   {% if debug %}RAISE WARNING 'Existing: %',existing.osm_id;{% endif %}
 
-  -- Handle a place changing type by removing the old data.
-  -- (This trigger is executed BEFORE INSERT of the NEW tuple.)
   IF existing.osm_type IS NULL THEN
     DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class;
   END IF;
@@ -187,15 +190,11 @@ BEGIN
     END IF;
     {% endif %}
 
-    IF existing.osm_type IS NOT NULL THEN
+    IF existingplacex.osm_type is not NULL THEN
-      -- Pathological case caused by the triggerless copy into place during initial import
+      -- Mark any existing place for delete in the placex table
-      -- force delete even for large areas, it will be reinserted later
+      UPDATE placex SET indexed_status = 100
-      UPDATE place SET geometry = ST_SetSRID(ST_Point(0,0), 4326)
+        WHERE placex.osm_type = NEW.osm_type and placex.osm_id = NEW.osm_id
-        WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
+              and placex.class = NEW.class and placex.type = NEW.type;
-              and class = NEW.class and type = NEW.type;
-      DELETE FROM place
-        WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
-              and class = NEW.class and type = NEW.type;
     END IF;
 
     -- Process it as a new insertion
@@ -206,6 +205,27 @@ BEGIN
 
     {% if debug %}RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name;{% endif %}
 
+    IF existing.osm_type is not NULL THEN
+      -- If there is already an entry in place, just update that, if necessary.
+      IF coalesce(existing.name, ''::hstore) != coalesce(NEW.name, ''::hstore)
+         or coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore)
+         or coalesce(existing.extratags, ''::hstore) != coalesce(NEW.extratags, ''::hstore)
+         or coalesce(existing.admin_level, 15) != coalesce(NEW.admin_level, 15)
+         or existing.geometry::text != NEW.geometry::text
+      THEN
+        UPDATE place
+          SET name = NEW.name,
+              address = NEW.address,
+              extratags = NEW.extratags,
+              admin_level = NEW.admin_level,
+              geometry = NEW.geometry
+          WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
+                and class = NEW.class and type = NEW.type;
+      END IF;
+
+      RETURN NULL;
+    END IF;
+
     RETURN NEW;
   END IF;
 
@@ -321,35 +341,67 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
-
 CREATE OR REPLACE FUNCTION place_delete()
   RETURNS TRIGGER
   AS $$
 DECLARE
-  has_rank BOOLEAN;
+  deferred BOOLEAN;
 BEGIN
+  {% if debug %}RAISE WARNING 'Delete for % % %/%', OLD.osm_type, OLD.osm_id, OLD.class, OLD.type;{% endif %}
 
-  {% if debug %}RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type;{% endif %}
+  deferred := ST_IsValid(OLD.geometry) and ST_Area(OLD.geometry) > 2;
-
+  IF deferred THEN
-  -- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
+    SELECT bool_or(not (rank_address = 0 or rank_address > 25)) INTO deferred
-  IF st_area(OLD.geometry) > 2 and st_isvalid(OLD.geometry) THEN
+      FROM placex
-    SELECT bool_or(not (rank_address = 0 or rank_address > 25)) as ranked FROM placex WHERE osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type INTO has_rank;
+      WHERE osm_type = OLD.osm_type and osm_id = OLD.osm_id
-    IF has_rank THEN
+            and class = OLD.class and type = OLD.type;
-      insert into import_polygon_delete (osm_type, osm_id, class, type) values (OLD.osm_type,OLD.osm_id,OLD.class,OLD.type);
-      RETURN NULL;
-    END IF;
   END IF;
 
-  -- mark for delete
+  INSERT INTO place_to_be_deleted (osm_type, osm_id, class, type, deferred)
-  UPDATE placex set indexed_status = 100 where osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type;
+    VALUES(OLD.osm_type, OLD.osm_id, OLD.class, OLD.type, deferred);
 
-  -- interpolations are special
+  RETURN NULL;
-  IF OLD.osm_type='W' and OLD.class = 'place' and OLD.type = 'houses' THEN
-    UPDATE location_property_osmline set indexed_status = 100 where osm_id = OLD.osm_id; -- osm_id = wayid (=old.osm_id)
-  END IF;
-
-  RETURN OLD;
 END;
-$$
+$$ LANGUAGE plpgsql;
-LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION flush_deleted_places()
+  RETURNS INTEGER
+  AS $$
+BEGIN
+  -- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
+  INSERT INTO import_polygon_delete (osm_type, osm_id, class, type)
+    SELECT osm_type, osm_id, class, type FROM place_to_be_deleted WHERE deferred;
+
+  -- delete from place table
+  ALTER TABLE place DISABLE TRIGGER place_before_delete;
+  DELETE FROM place USING place_to_be_deleted
+    WHERE place.osm_type = place_to_be_deleted.osm_type
+          and place.osm_id = place_to_be_deleted.osm_id
+          and place.class = place_to_be_deleted.class
+          and place.type = place_to_be_deleted.type
+          and not deferred;
+  ALTER TABLE place ENABLE TRIGGER place_before_delete;
+
+  -- Mark for delete in the placex table
+  UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
+    WHERE placex.osm_type = place_to_be_deleted.osm_type
+          and placex.osm_id = place_to_be_deleted.osm_id
+          and placex.class = place_to_be_deleted.class
+          and placex.type = place_to_be_deleted.type
+          and not deferred;
+
+   -- Mark for delete in interpolations
+   UPDATE location_property_osmline SET indexed_status = 100 FROM place_to_be_deleted
+    WHERE place_to_be_deleted.osm_type = 'W'
+          and place_to_be_deleted.class = 'place'
+          and place_to_be_deleted.type = 'houses'
+          and location_property_osmline.osm_id = place_to_be_deleted.osm_id
+          and not deferred;
+
+   -- Clear todo list.
+   TRUNCATE TABLE place_to_be_deleted;
+
+   RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
 

lib-sql/indices.sql

@@ -82,4 +82,14 @@ CREATE INDEX IF NOT EXISTS idx_postcode_postcode
       INCLUDE (startnumber, endnumber) {{db.tablespace.search_index}}
       WHERE startnumber is not null;
   {% endif %}
+---
+-- Table needed for running updates with osm2pgsql on place.
+  CREATE TABLE IF NOT EXISTS place_to_be_deleted (
+    osm_type CHAR(1),
+    osm_id BIGINT,
+    class TEXT,
+    type TEXT,
+    deferred BOOLEAN
+   );
+
 {% endif %}

nominatim/clicmd/args.py

@@ -184,6 +184,7 @@ class NominatimArgs:
         return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.osm2pgsql_path,
                     osm2pgsql_cache=self.osm2pgsql_cache or default_cache,
                     osm2pgsql_style=self.config.get_import_style_file(),
+                    osm2pgsql_style_path=self.config.config_dir,
                     threads=self.threads or default_threads,
                     dsn=self.config.get_libpq_dsn(),
                     flatnode_file=str(self.config.get_path('FLATNODE_FILE') or ''),

nominatim/tools/exec_utils.py

@@ -10,6 +10,7 @@ Helper functions for executing external programs.
 from typing import Any, Union, Optional, Mapping, IO
 from pathlib import Path
 import logging
+import os
 import subprocess
 import urllib.request as urlrequest
 from urllib.parse import urlencode
@@ -120,9 +121,16 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
            '--log-progress', 'true',
            '--number-processes', str(options['threads']),
            '--cache', str(options['osm2pgsql_cache']),
-           '--output', 'gazetteer',
            '--style', str(options['osm2pgsql_style'])
           ]
+
+    if str(options['osm2pgsql_style']).endswith('.lua'):
+        env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / 'flex-base.lua'),
+                                    os.environ.get('LUAPATH', ';')))
+        cmd.extend(('--output', 'flex'))
+    else:
+        cmd.extend(('--output', 'gazetteer'))
+
     if options['append']:
         cmd.append('--append')
     else:

nominatim/tools/migration.py

@@ -315,3 +315,20 @@ def mark_internal_country_names(conn: Connection, config: Configuration, **_: An
                     names = {}
                 names['countrycode'] = country_code
                 analyzer.add_country_names(country_code, names)
+
+
+@_migration(4, 1, 99, 0)
+def add_place_deletion_todo_table(conn: Connection, **_: Any) -> None:
+    """ Add helper table for deleting data on updates.
+
+        The table is only necessary when updates are possible, i.e.
+        the database is not in freeze mode.
+    """
+    if conn.table_exists('place'):
+        with conn.cursor() as cur:
+            cur.execute("""CREATE TABLE IF NOT EXISTS place_to_be_deleted (
+                             osm_type CHAR(1),
+                             osm_id BIGINT,
+                             class TEXT,
+                             type TEXT,
+                             deferred BOOLEAN)""")

nominatim/tools/replication.py

@@ -130,10 +130,7 @@ def update(conn: Connection, options: MutableMapping[str, Any],
         if endseq is None:
             return UpdateState.NO_CHANGES
 
-        # Consume updates with osm2pgsql.
+        run_osm2pgsql_updates(conn, options)
-        options['append'] = True
-        options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
-        run_osm2pgsql(options)
 
         # Write the current status to the file
         endstate = repl.get_state_info(endseq)
@@ -143,6 +140,25 @@ def update(conn: Connection, options: MutableMapping[str, Any],
     return UpdateState.UP_TO_DATE
 
 
+def run_osm2pgsql_updates(conn: Connection, options: MutableMapping[str, Any]) -> None:
+    """ Run osm2pgsql in append mode.
+    """
+    # Remove any stale deletion marks.
+    with conn.cursor() as cur:
+        cur.execute('TRUNCATE place_to_be_deleted')
+    conn.commit()
+
+    # Consume updates with osm2pgsql.
+    options['append'] = True
+    options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
+    run_osm2pgsql(options)
+
+    # Handle deletions
+    with conn.cursor() as cur:
+        cur.execute('SELECT flush_deleted_places()')
+    conn.commit()
+
+
 def _make_replication_server(url: str, timeout: int) -> ContextManager[ReplicationServer]:
     """ Returns a ReplicationServer in form of a context manager.
 

nominatim/version.py

@@ -25,7 +25,7 @@ from typing import Optional, Tuple
 # patch level when cherry-picking the commit with the migration.
 #
 # Released versions always have a database patch level of 0.
-NOMINATIM_VERSION = (4, 1, 0, 0)
+NOMINATIM_VERSION = (4, 1, 99, 0)
 
 POSTGRESQL_REQUIRED_VERSION = (9, 6)
 POSTGIS_REQUIRED_VERSION = (2, 2)

settings/flex-base.lua

@@ -0,0 +1,393 @@
+-- Core functions for Nominatim import flex style.
+--
+
+
+-- The single place table.
+place_table = osm2pgsql.define_table{
+    name = "place",
+    ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
+    columns = {
+        { column = 'class', type = 'text', not_null = true },
+        { column = 'type', type = 'text', not_null = true },
+        { column = 'admin_level', type = 'smallint' },
+        { column = 'name', type = 'hstore' },
+        { column = 'address', type = 'hstore' },
+        { column = 'extratags', type = 'hstore' },
+        { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
+    }
+}
+
+------------- Place class ------------------------------------------
+
+local Place = {}
+Place.__index = Place
+
+function Place.new(object, geom_func)
+    local self = setmetatable({}, Place)
+    self.object = object
+    self.geom_func = geom_func
+
+    self.admin_level = tonumber(self.object:grab_tag('admin_level'))
+    if self.admin_level == nil
+       or self.admin_level <= 0 or self.admin_level > 15
+       or math.floor(self.admin_level) ~= self.admin_level then
+        self.admin_level = 15
+    end
+
+    self.num_entries = 0
+    self.has_name = false
+    self.names = {}
+    self.address = {}
+    self.extratags = {}
+
+    return self
+end
+
+function Place:delete(data)
+    if data.match ~= nil then
+        for k, v in pairs(self.object.tags) do
+            if data.match(k, v) then
+                self.object.tags[k] = nil
+            end
+        end
+    end
+end
+
+function Place:grab_extratags(data)
+    local count = 0
+
+    if data.match ~= nil then
+        for k, v in pairs(self.object.tags) do
+            if data.match(k, v) then
+                self.object.tags[k] = nil
+                self.extratags[k] = v
+                count = count + 1
+            end
+        end
+    end
+
+    return count
+end
+
+function Place:grab_address(data)
+    local count = 0
+
+    if data.match ~= nil then
+        for k, v in pairs(self.object.tags) do
+            if data.match(k, v) then
+                self.object.tags[k] = nil
+
+                if data.include_on_name == true then
+                    self.has_name = true
+                end
+
+                if data.out_key ~= nil then
+                    self.address[data.out_key] = v
+                    return 1
+                end
+
+                if k:sub(1, 5) == 'addr:' then
+                    self.address[k:sub(6)] = v
+                elseif k:sub(1, 6) == 'is_in:' then
+                    self.address[k:sub(7)] = v
+                else
+                    self.address[k] = v
+                end
+                count = count + 1
+            end
+        end
+    end
+
+    return count
+end
+
+function Place:set_address(key, value)
+    self.address[key] = value
+end
+
+function Place:grab_name(data)
+    local count = 0
+
+    if data.match ~= nil then
+        for k, v in pairs(self.object.tags) do
+            if data.match(k, v) then
+                self.object.tags[k] = nil
+                self.names[k] = v
+                if data.include_on_name ~= false then
+                    self.has_name = true
+                end
+                count = count + 1
+            end
+        end
+    end
+
+    return count
+end
+
+function Place:grab_tag(key)
+    return self.object:grab_tag(key)
+end
+
+function Place:tags()
+    return self.object.tags
+end
+
+function Place:write_place(k, v, mtype, save_extra_mains)
+    if mtype == nil then
+        return 0
+    end
+
+    v = v or self.object.tags[k]
+    if v == nil then
+        return 0
+    end
+
+    if type(mtype) == 'table' then
+        mtype = mtype[v] or mtype[1]
+    end
+
+    if mtype == 'always' or (self.has_name and mtype == 'named') then
+        return self:write_row(k, v, save_extra_mains)
+    end
+
+    if mtype == 'named_with_key' then
+        local names = {}
+        local prefix = k .. ':name'
+        for namek, namev in pairs(self.object.tags) do
+            if namek:sub(1, #prefix) == prefix
+               and (#namek == #prefix
+                    or namek:sub(#prefix + 1, #prefix + 1) == ':') then
+                names[namek:sub(#k + 2)] = namev
+            end
+        end
+
+        if next(names) ~= nil then
+            local saved_names = self.names
+            self.names = names
+
+            local results = self:write_row(k, v, save_extra_mains)
+
+            self.names = saved_names
+
+            return results
+        end
+    end
+
+    return 0
+end
+
+function Place:write_row(k, v, save_extra_mains)
+    if self.geometry == nil then
+        self.geometry = self.geom_func(self.object)
+    end
+    if self.geometry:is_null() then
+        return 0
+    end
+
+    if save_extra_mains then
+        for extra_k, extra_v in pairs(self.object.tags) do
+            if extra_k ~= k then
+                self.extratags[extra_k] = extra_v
+            end
+        end
+    end
+
+    place_table:insert{
+        class = k,
+        type = v,
+        admin_level = self.admin_level,
+        name = next(self.names) and self.names,
+        address = next(self.address) and self.address,
+        extratags = next(self.extratags) and self.extratags,
+        geometry = self.geometry
+    }
+
+    if save_extra_mains then
+        for k, v in pairs(self.object.tags) do
+            self.extratags[k] = nil
+        end
+    end
+
+    self.num_entries = self.num_entries + 1
+
+    return 1
+end
+
+
+function tag_match(data)
+    if data == nil or next(data) == nil then
+        return nil
+    end
+
+    local fullmatches = {}
+    local key_prefixes = {}
+    local key_suffixes = {}
+
+    if data.keys ~= nil then
+        for _, key in pairs(data.keys) do
+            if key:sub(1, 1) == '*' then
+                if #key > 1 then
+                    if key_suffixes[#key - 1] == nil then
+                        key_suffixes[#key - 1] = {}
+                    end
+                    key_suffixes[#key - 1][key:sub(2)] = true
+                end
+            elseif key:sub(#key, #key) == '*' then
+                if key_prefixes[#key - 1] == nil then
+                    key_prefixes[#key - 1] = {}
+                end
+                key_prefixes[#key - 1][key:sub(1, #key - 1)] = true
+            else
+                fullmatches[key] = true
+            end
+        end
+    end
+
+    if data.tags ~= nil then
+        for k, vlist in pairs(data.tags) do
+            if fullmatches[k] == nil then
+                fullmatches[k] = {}
+                for _, v in pairs(vlist) do
+                    fullmatches[k][v] = true
+                end
+            end
+        end
+    end
+
+    return function (k, v)
+        if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then
+            return true
+        end
+
+        for slen, slist in pairs(key_suffixes) do
+            if #k >= slen and slist[k:sub(-slen)] ~= nil then
+                return true
+            end
+        end
+
+        for slen, slist in pairs(key_prefixes) do
+            if #k >= slen and slist[k:sub(1, slen)] ~= nil then
+                return true
+            end
+        end
+
+        return false
+    end
+end
+
+
+-- Process functions for all data types
+function osm2pgsql.process_node(object)
+
+    local function geom_func(o)
+        return o:as_point()
+    end
+
+    process_tags(Place.new(object, geom_func))
+end
+
+function osm2pgsql.process_way(object)
+
+    local function geom_func(o)
+        local geom = o:as_polygon()
+
+        if geom:is_null() then
+            geom = o:as_linestring()
+        end
+
+        return geom
+    end
+
+    process_tags(Place.new(object, geom_func))
+end
+
+function relation_as_multipolygon(o)
+    return o:as_multipolygon()
+end
+
+function relation_as_multiline(o)
+    return o:as_multilinestring():line_merge()
+end
+
+function osm2pgsql.process_relation(object)
+    local geom_func = RELATION_TYPES[object.tags.type]
+
+    if geom_func ~= nil then
+        process_tags(Place.new(object, geom_func))
+    end
+end
+
+function process_tags(o)
+    local fallback
+
+    o:delete{match = PRE_DELETE}
+    o:grab_extratags{match = PRE_EXTRAS}
+
+    -- Exception for boundary/place double tagging
+    if o.object.tags.boundary == 'administrative' then
+        o:grab_extratags{match = function (k, v)
+            return k == 'place' and v:sub(1,3) ~= 'isl'
+        end}
+    end
+
+    -- address keys
+    o:grab_address{match=COUNTRY_TAGS, out_key='country'}
+    if o.address.country ~= nil and #o.address.country ~= 2 then
+        o.address['country'] = nil
+    end
+    if o:grab_name{match=HOUSENAME_TAGS} > 0 then
+        fallback = {'place', 'house'}
+    end
+    if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then
+        fallback = {'place', 'house'}
+    end
+    if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then
+        fallback = {'place', 'postcode'}
+    end
+
+    local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0
+
+    if ADD_TIGER_COUNTY then
+        local v = o:grab_tag('tiger:county')
+        if v ~= nil then
+            v, num = v:gsub(',.*', ' county')
+            if num == 0 then
+                v = v .. ' county'
+            end
+            o:set_address('tiger:county', v)
+        end
+    end
+    o:grab_address{match=ADDRESS_TAGS}
+
+    if is_interpolation then
+        o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
+        return
+    end
+
+    -- name keys
+    o:grab_name{match = NAMES}
+    o:grab_name{match = REFS, include_on_name = false}
+
+    o:delete{match = POST_DELETE}
+    o:grab_extratags{match = POST_EXTRAS}
+
+    -- collect main keys
+    local num_mains = 0
+    for k, v in pairs(o:tags()) do
+        num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
+    end
+
+    if num_mains == 0 then
+        for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do
+            if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then
+                return
+            end
+        end
+
+        if fallback ~= nil then
+            o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS)
+        end
+    end
+end
+
+

settings/import-extratags.lua

@@ -0,0 +1,130 @@
+require('flex-base')
+
+RELATION_TYPES = {
+    multipolygon = relation_as_multipolygon,
+    boundary = relation_as_multipolygon,
+    waterway = relation_as_multiline
+}
+
+MAIN_KEYS = {
+    emergency = 'always',
+    historic = 'always',
+    military = 'always',
+    natural = 'named',
+    landuse = 'named',
+    highway = {'always',
+               street_lamp = 'named',
+               traffic_signals = 'named',
+               service = 'named',
+               cycleway = 'named',
+               path = 'named',
+               footway = 'named',
+               steps = 'named',
+               bridleway = 'named',
+               track = 'named',
+               motorway_link = 'named',
+               trunk_link = 'named',
+               primary_link = 'named',
+               secondary_link = 'named',
+               tertiary_link = 'named'},
+    railway = 'named',
+    man_made = 'always',
+    aerialway = 'always',
+    boundary = {'named',
+                postal_code = 'named'},
+    aeroway = 'always',
+    amenity = 'always',
+    club = 'always',
+    craft = 'always',
+    leisure = 'always',
+    office = 'always',
+    mountain_pass = 'always',
+    shop = 'always',
+    tourism = 'always',
+    bridge = 'named_with_key',
+    tunnel = 'named_with_key',
+    waterway = 'named',
+    place = 'always'
+}
+
+MAIN_FALLBACK_KEYS = {
+    building = 'named',
+    landuse = 'named',
+    junction = 'named',
+    healthcare = 'named'
+}
+
+
+PRE_DELETE = tag_match{keys = {'note', 'note:*', 'source', 'source*', 'attribution',
+                               'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
+                               'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
+                               'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
+                               'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
+                               'ref:linz:*', 'is_in:postcode'},
+                       tags = {emergency = {'yes', 'no', 'fire_hydrant'},
+                               historic = {'yes', 'no'},
+                               military = {'yes', 'no'},
+                               natural = {'yes', 'no', 'coastline'},
+                               highway = {'no', 'turning_circle', 'mini_roundabout',
+                                          'noexit', 'crossing', 'give_way', 'stop'},
+                               railway = {'level_crossing', 'no', 'rail'},
+                               man_made = {'survey_point', 'cutline'},
+                               aerialway = {'pylon', 'no'},
+                               aeroway = {'no'},
+                               amenity = {'no'},
+                               club = {'no'},
+                               craft = {'no'},
+                               leisure = {'no'},
+                               office = {'no'},
+                               mountain_pass = {'no'},
+                               shop = {'no'},
+                               tourism = {'yes', 'no'},
+                               bridge = {'no'},
+                               tunnel = {'no'},
+                               waterway = {'riverbank'},
+                               building = {'no'},
+                               boundary = {'place'}}
+                      }
+
+POST_DELETE = tag_match{keys = {'tiger:*'}}
+
+PRE_EXTRAS = tag_match{keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                               'name:etymology', 'name:signed', 'name:botanical',
+                               'wikidata', '*:wikidata',
+                               'addr:street:name', 'addr:street:type'}
+                      }
+
+
+NAMES = tag_match{keys = {'name', 'name:*',
+                          'int_name', 'int_name:*',
+                          'nat_name', 'nat_name:*',
+                          'reg_name', 'reg_name:*',
+                          'loc_name', 'loc_name:*',
+                          'old_name', 'old_name:*',
+                          'alt_name', 'alt_name:*', 'alt_name_*',
+                          'official_name', 'official_name:*',
+                          'place_name', 'place_name:*',
+                          'short_name', 'short_name:*', 'brand'}}
+
+REFS = tag_match{keys = {'ref', 'int_ref', 'nat_ref', 'reg_ref', 'loc_ref', 'old_ref',
+                         'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}}
+
+POSTCODES = tag_match{keys = {'postal_code', 'postcode', 'addr:postcode',
+                              'tiger:zip_left', 'tiger:zip_right'}}
+
+COUNTRY_TAGS = tag_match{keys = {'country_code', 'ISO3166-1',
+                                 'addr:country_code', 'is_in:country_code',
+                                 'addr:country', 'is_in:country'}}
+
+HOUSENAME_TAGS = tag_match{keys = {'addr:housename'}}
+
+HOUSENUMBER_TAGS = tag_match{keys = {'addr:housenumber', 'addr:conscriptionnumber',
+                                     'addr:streetnumber'}}
+
+INTERPOLATION_TAGS = tag_match{keys = {'addr:interpolation'}}
+
+ADDRESS_TAGS = tag_match{keys = {'addr:*', 'is_in:*'}}
+ADD_TIGER_COUNTY = true
+
+SAVE_EXTRA_MAINS = true
+

test/bdd/environment.py

@@ -27,6 +27,7 @@ userconfig = {
     'API_TEST_FILE'  : (TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf').resolve(),
     'SERVER_MODULE_PATH' : None,
     'TOKENIZER' : None, # Test with a custom tokenizer
+    'STYLE' : 'extratags',
     'PHPCOV' : False, # set to output directory to enable code coverage
 }
 

test/bdd/osm2pgsql/import/tags.feature

@@ -0,0 +1,209 @@
+@DB
+Feature: Tag evaluation
+    Tests if tags are correctly imported into the place table
+
+    Scenario: Main tags as fallback
+        When loading osm data
+            """
+            n100 Tjunction=yes,highway=bus_stop
+            n101 Tjunction=yes,name=Bar
+            n200 Tbuilding=yes,amenity=cafe
+            n201 Tbuilding=yes,name=Intersting
+            n202 Tbuilding=yes
+            """
+        Then place contains exactly
+            | object | class    | type     |
+            | N100   | highway  | bus_stop |
+            | N101   | junction | yes      |
+            | N200   | amenity  | cafe     |
+            | N201   | building | yes      |
+
+
+    Scenario: Name and reg tags
+        When loading osm data
+            """
+            n2001 Thighway=road,name=Foo,alt_name:de=Bar,ref=45
+            n2002 Thighway=road,name:prefix=Pre,name:suffix=Post,ref:de=55
+            n2003 Thighway=yes,name:%20%de=Foo,name=real1
+            n2004 Thighway=yes,name:%a%de=Foo,name=real2
+            n2005 Thighway=yes,name:%9%de=Foo,name:\\=real3
+            n2006 Thighway=yes,name:%9%de=Foo,name=rea\l3
+            """
+        Then place contains exactly
+            | object | class   | type | name |
+            | N2001  | highway | road | 'name': 'Foo', 'alt_name:de': 'Bar', 'ref': '45' |
+            | N2002  | highway | road | - |
+            | N2003  | highway | yes  | 'name: de': 'Foo', 'name': 'real1' |
+            | N2004  | highway | yes  | 'name:\nde': 'Foo', 'name': 'real2' |
+            | N2005  | highway | yes  | 'name:\tde': 'Foo', 'name:\\\\': 'real3' |
+            | N2006  | highway | yes  | 'name:\tde': 'Foo', 'name': 'rea\\l3' |
+
+        And place contains
+            | object | extratags |
+            | N2002  | 'name:prefix': 'Pre', 'name:suffix': 'Post', 'ref:de': '55' |
+
+
+    Scenario: Name when using with_name flag
+        When loading osm data
+            """
+            n3001 Tbridge=yes,bridge:name=GoldenGate
+            n3002 Tbridge=yes,bridge:name:en=Rainbow
+            """
+        Then place contains exactly
+            | object | class   | type | name                 |
+            | N3001  | bridge  | yes  | 'name': 'GoldenGate' |
+            | N3002  | bridge  | yes  | 'name:en': 'Rainbow' |
+
+
+    Scenario: Address tags
+        When loading osm data
+            """
+            n4001 Taddr:housenumber=34,addr:city=Esmarald,addr:county=Land
+            n4002 Taddr:streetnumber=10,is_in:city=Rootoo,is_in=Gold
+            """
+        Then place contains exactly
+            | object | class | address             |
+            | N4001  | place | 'housenumber': '34', 'city': 'Esmarald', 'county': 'Land' |
+            | N4002  | place | 'streetnumber': '10', 'city': 'Rootoo' |
+
+
+    Scenario: Country codes
+        When loading osm data
+            """
+            n5001 Tshop=yes,country_code=DE
+            n5002 Tshop=yes,country_code=toolong
+            n5003 Tshop=yes,country_code=x
+            n5004 Tshop=yes,addr:country=us
+            n5005 Tshop=yes,country=be
+            n5006 Tshop=yes,addr:country=France
+            """
+        Then place contains exactly
+            | object | class | address         |
+            | N5001  | shop  | 'country': 'DE' |
+            | N5002  | shop  | - |
+            | N5003  | shop  | - |
+            | N5004  | shop  | 'country': 'us' |
+            | N5005  | shop  | - |
+            | N5006  | shop  | - |
+
+
+    Scenario: Postcodes
+        When loading osm data
+            """
+            n6001 Tshop=bank,addr:postcode=12345
+            n6002 Tshop=bank,tiger:zip_left=34343
+            n6003 Tshop=bank,is_in:postcode=9009
+            """
+        Then place contains exactly
+            | object | class | address             |
+            | N6001  | shop  | 'postcode': '12345' |
+            | N6002  | shop  | 'postcode': '34343' |
+            | N6003  | shop  | -                   |
+
+
+    Scenario: Main with extra
+        When loading osm data
+            """
+            n7001 Thighway=primary,bridge=yes,name=1
+            n7002 Thighway=primary,bridge=yes,bridge:name=1
+            """
+        Then place contains exactly
+            | object        | class   | type    | name        | extratags+bridge:name |
+            | N7001         | highway | primary | 'name': '1' | -                     |
+            | N7002:highway | highway | primary | -           | 1                     |
+            | N7002:bridge  | bridge  | yes     | 'name': '1' | 1                     |
+
+
+    Scenario: Global fallback and skipping
+        When loading osm data
+            """
+            n8001 Tshop=shoes,note:de=Nein,xx=yy
+            n8002 Tshop=shoes,building=no,ele=234
+            n8003 Tshop=shoes,name:source=survey
+            """
+        Then place contains exactly
+            | object | class | extratags    |
+            | N8001  | shop  | 'xx': 'yy'   |
+            | N8002  | shop  | 'ele': '234' |
+            | N8003  | shop  | -            |
+
+
+    Scenario: Admin levels
+        When loading osm data
+            """
+            n9001 Tplace=city
+            n9002 Tplace=city,admin_level=16
+            n9003 Tplace=city,admin_level=x
+            n9004 Tplace=city,admin_level=1
+            n9005 Tplace=city,admin_level=0
+            n9006 Tplace=city,admin_level=2.5
+            """
+        Then place contains exactly
+            | object | class | admin_level |
+            | N9001  | place | 15          |
+            | N9002  | place | 15          |
+            | N9003  | place | 15          |
+            | N9004  | place | 1           |
+            | N9005  | place | 15          |
+            | N9006  | place | 15          |
+
+
+    Scenario: Administrative boundaries with place tags
+        When loading osm data
+            """
+            n10001 Tboundary=administrative,place=city,name=A
+            n10002 Tboundary=natural,place=city,name=B
+            n10003 Tboundary=administrative,place=island,name=C
+            """
+        Then place contains
+            | object          | class    | type           | extratags       |
+            | N10001          | boundary | administrative | 'place': 'city' |
+        And place contains
+            | object          | class    | type           |
+            | N10002:boundary | boundary | natural        |
+            | N10002:place    | place    | city           |
+            | N10003:boundary | boundary | administrative |
+            | N10003:place    | place    | island         |
+
+
+    Scenario: Shorten tiger:county tags
+        When loading osm data
+            """
+            n11001 Tplace=village,tiger:county=Feebourgh%2c%%20%AL
+            n11002 Tplace=village,addr:state=Alabama,tiger:county=Feebourgh%2c%%20%AL
+            n11003 Tplace=village,tiger:county=Feebourgh
+            """
+        Then place contains exactly
+            | object | class | address             |
+            | N11001 | place | 'tiger:county': 'Feebourgh county' |
+            | N11002 | place | 'tiger:county': 'Feebourgh county', 'state': 'Alabama' |
+            | N11003 | place | 'tiger:county': 'Feebourgh county' |
+
+
+    Scenario: Building fallbacks
+        When loading osm data
+            """
+            n12001 Ttourism=hotel,building=yes
+            n12002 Tbuilding=house
+            n12003 Tbuilding=shed,addr:housenumber=1
+            n12004 Tbuilding=yes,name=Das-Haus
+            n12005 Tbuilding=yes,addr:postcode=12345
+            """
+        Then place contains exactly
+            | object | class    | type     |
+            | N12001 | tourism  | hotel    |
+            | N12003 | building | shed     |
+            | N12004 | building | yes      |
+            | N12005 | place    | postcode |
+
+
+    Scenario: Address interpolations
+        When loading osm data
+            """
+            n13001 Taddr:interpolation=odd
+            n13002 Taddr:interpolation=even,place=city
+            """
+        Then place contains exactly
+            | object | class | type   | address                 |
+            | N13001 | place | houses | 'interpolation': 'odd'  |
+            | N13002 | place | houses | 'interpolation': 'even' |

test/bdd/osm2pgsql/update/interpolations.feature

@@ -0,0 +1,133 @@
+@DB
+Feature: Updates of address interpolation objects
+    Test that changes to address interpolation objects are correctly
+    propagated.
+
+    Background:
+        Given the grid
+            | 1 | 2 |
+
+
+    Scenario: Adding a new interpolation
+        When loading osm data
+            """
+            n1 Taddr:housenumber=3
+            n2 Taddr:housenumber=17
+            """
+        Then place contains
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+
+        When updating osm data
+            """
+            w99 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+            | W99:place | houses |
+        When indexing
+        Then placex contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+        Then location_property_osmline contains exactly
+            | object |
+            | 99:5   |
+
+
+    Scenario: Delete an existing interpolation
+        When loading osm data
+            """
+            n1 Taddr:housenumber=2
+            n2 Taddr:housenumber=7
+            w99 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+            | W99:place | houses |
+
+        When updating osm data
+            """
+            w99 v2 dD
+            """
+        Then place contains
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+        When indexing
+        Then placex contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+        Then location_property_osmline contains exactly
+            | object | indexed_status |
+
+
+    Scenario: Changing an object to an interpolation
+        When loading osm data
+            """
+            n1 Taddr:housenumber=3
+            n2 Taddr:housenumber=17
+            w99 Thighway=residential Nn1,n2
+            """
+        Then place contains
+            | object      | type   |
+            | N1:place    | house  |
+            | N2:place    | house  |
+            | W99:highway | residential  |
+
+        When updating osm data
+            """
+            w99 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+            | W99:place | houses |
+        When indexing
+        Then placex contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+        And location_property_osmline contains exactly
+            | object |
+            | 99:5   |
+
+
+    Scenario: Changing an interpolation to something else
+        When loading osm data
+            """
+            n1 Taddr:housenumber=3
+            n2 Taddr:housenumber=17
+            w99 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains
+            | object      | type   |
+            | N1:place    | house  |
+            | N2:place    | house  |
+            | W99:place | houses |
+
+        When updating osm data
+            """
+            w99 Thighway=residential Nn1,n2
+            """
+        Then place contains
+            | object      | type   |
+            | N1:place    | house  |
+            | N2:place    | house  |
+            | W99:highway | residential  |
+        When indexing
+        Then placex contains exactly
+            | object      | type   |
+            | N1:place    | house  |
+            | N2:place    | house  |
+            | W99:highway | residential  |
+        And location_property_osmline contains exactly
+            | object |
+

test/bdd/osm2pgsql/update/postcodes.feature

@@ -0,0 +1,163 @@
+@DB
+Feature: Update of postcode only objects
+    Tests that changes to objects containing only a postcode are
+    propagated correctly.
+
+
+    Scenario: Adding a postcode-only node
+        When loading osm data
+            """
+            """
+        Then place contains exactly
+            | object |
+
+        When updating osm data
+            """
+            n34 Tpostcode=4456
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N34:place | postcode |
+        When indexing
+        Then placex contains exactly
+            | object |
+
+
+    Scenario: Deleting a postcode-only node
+        When loading osm data
+            """
+            n34 Tpostcode=4456
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N34:place | postcode |
+
+        When updating osm data
+            """
+            n34 v2 dD
+            """
+        Then place contains exactly
+            | object |
+        When indexing
+        Then placex contains exactly
+            | object |
+
+
+    Scenario Outline: Converting a regular object into a postcode-only node
+        When loading osm data
+            """
+            n34 T<class>=<type>
+            """
+        Then place contains exactly
+            | object      | type   |
+            | N34:<class> | <type> |
+
+        When updating osm data
+            """
+            n34 Tpostcode=4456
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N34:place | postcode |
+        When indexing
+        Then placex contains exactly
+            | object |
+
+        Examples:
+            | class   | type       |
+            | amenity | restaurant |
+            | place   | hamlet     |
+
+
+    Scenario Outline: Converting a postcode-only node into a regular object
+        When loading osm data
+            """
+            n34 Tpostcode=4456
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N34:place | postcode |
+
+        When updating osm data
+            """
+            n34 T<class>=<type>
+            """
+        Then place contains exactly
+            | object      | type   |
+            | N34:<class> | <type> |
+        When indexing
+        Then placex contains exactly
+            | object      | type   |
+            | N34:<class> | <type> |
+
+        Examples:
+            | class   | type       |
+            | amenity | restaurant |
+            | place   | hamlet     |
+
+
+    Scenario: Converting na interpolation into a postcode-only node
+        Given the grid
+            | 1 | 2 |
+        When loading osm data
+            """
+            n1 Taddr:housenumber=3
+            n2 Taddr:housenumber=17
+            w34 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+            | W34:place | houses |
+
+        When updating osm data
+            """
+            w34 Tpostcode=4456 Nn1,n2
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N1:place  | house    |
+            | N2:place  | house    |
+            | W34:place | postcode |
+        When indexing
+        Then location_property_osmline contains exactly
+            | object |
+        And placex contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+
+
+    Scenario: Converting a postcode-only node into an interpolation
+        Given the grid
+            | 1 | 2 |
+        When loading osm data
+            """
+            n1 Taddr:housenumber=3
+            n2 Taddr:housenumber=17
+            w34 Tpostcode=4456 Nn1,n2
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N1:place  | house    |
+            | N2:place  | house    |
+            | W34:place | postcode |
+
+        When updating osm data
+            """
+            w34 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+            | W34:place | houses |
+        When indexing
+        Then location_property_osmline contains exactly
+            | object |
+            | 34:5   |
+        And placex contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |

test/bdd/osm2pgsql/update/simple.feature

@@ -2,60 +2,48 @@
 Feature: Update of simple objects by osm2pgsql
     Testing basic update functions of osm2pgsql.
 
-    Scenario: Import object with two main tags
+    Scenario: Adding a new object
         When loading osm data
           """
-          n1 Ttourism=hotel,amenity=restaurant,name=foo
+          n1 Tplace=town,name=Middletown
-          n2 Tplace=locality,name=spotty
           """
-        Then place contains
+        Then place contains exactly
-          | object     | type       | name+name |
+          | object   | type | name+name  |
-          | N1:tourism | hotel      | foo |
+          | N1:place | town | Middletown |
-          | N1:amenity | restaurant | foo |
-          | N2:place   | locality   | spotty |
-        When updating osm data
-          """
-          n1 dV Ttourism=hotel,name=foo
-          n2 dD
-          """
-        Then place has no entry for N1:amenity
-        And place has no entry for N2
-        And place contains
-          | object     | class   | type       | name |
-          | N1:tourism | tourism | hotel      | 'name' : 'foo' |
 
-    Scenario: Downgrading a highway to one that is dropped without name
+       When updating osm data
-        When loading osm data
+         """
-          """
+         n2 Tamenity=hotel,name=Posthotel
-          n100 x0 y0
+         """
-          n101 x0.0001 y0.0001
+        Then place contains exactly
-          w1 Thighway=residential Nn100,n101
+          | object     | type  | name+name  |
-          """
+          | N1:place   | town  | Middletown |
-        Then place contains
+          | N2:amenity | hotel | Posthotel  |
-          | object     |
+        And placex contains exactly
-          | W1:highway |
+          | object     | type  | name+name  | indexed_status |
-        When updating osm data
+          | N1:place   | town  | Middletown | 0              |
-          """
+          | N2:amenity | hotel | Posthotel  | 1              |
-          w1 Thighway=service Nn100,n101
-          """
-        Then place has no entry for W1
 
-    Scenario: Downgrading a highway when a second tag is present
+
+    Scenario: Deleting an existing object
         When loading osm data
           """
-          n100 x0 y0
+          n1 Tplace=town,name=Middletown
-          n101 x0.0001 y0.0001
+          n2 Tamenity=hotel,name=Posthotel
-          w1 Thighway=residential,tourism=hotel Nn100,n101
           """
-        Then place contains
+        Then place contains exactly
-          | object     |
+          | object     | type  | name+name  |
-          | W1:highway |
+          | N1:place   | town  | Middletown |
-          | W1:tourism |
+          | N2:amenity | hotel | Posthotel  |
-        When updating osm data
+
-          """
+       When updating osm data
-          w1 Thighway=service,tourism=hotel Nn100,n101
+         """
-          """
+         n2 dD
-        Then place has no entry for W1:highway
+         """
-        And place contains
+        Then place contains exactly
-          | object     |
+          | object     | type  | name+name  |
-          | W1:tourism |
+          | N1:place   | town  | Middletown |
+        And placex contains exactly
+          | object     | type  | name+name  | indexed_status |
+          | N1:place   | town  | Middletown | 0              |
+          | N2:amenity | hotel | Posthotel  | 100            |

test/bdd/osm2pgsql/update/tags.feature

@@ -0,0 +1,490 @@
+@DB
+Feature: Tag evaluation
+    Tests if tags are correctly updated in the place table
+
+    Background:
+        Given the grid
+            | 1  | 2  | 3 |
+            | 10 | 11 |   |
+            | 45 | 46 |   |
+
+    Scenario: Main tag deleted
+        When loading osm data
+            """
+            n1 Tamenity=restaurant
+            n2 Thighway=bus_stop,railway=stop,name=X
+            n3 Tamenity=prison
+            """
+        Then place contains exactly
+            | object     | class   | type       |
+            | N1         | amenity | restaurant |
+            | N2:highway | highway | bus_stop   |
+            | N2:railway | railway | stop       |
+            | N3         | amenity | prison     |
+
+        When updating osm data
+            """
+            n1 Tnot_a=restaurant
+            n2 Thighway=bus_stop,name=X
+            """
+        Then place contains exactly
+            | object     | class   | type       |
+            | N2:highway | highway | bus_stop   |
+            | N3         | amenity | prison     |
+        And placex contains
+            | object     | indexed_status |
+            | N3:amenity | 0              |
+        When indexing
+        Then placex contains exactly
+            | object     | type     | name        |
+            | N2:highway | bus_stop | 'name': 'X' |
+            | N3:amenity | prison   | -           |
+
+
+    Scenario: Main tag added
+        When loading osm data
+            """
+            n1 Tatity=restaurant
+            n2 Thighway=bus_stop,name=X
+            """
+        Then place contains exactly
+            | object     | class   | type       |
+            | N2:highway | highway | bus_stop   |
+
+        When updating osm data
+            """
+            n1 Tamenity=restaurant
+            n2 Thighway=bus_stop,railway=stop,name=X
+            """
+        Then place contains exactly
+            | object     | class   | type       |
+            | N1         | amenity | restaurant |
+            | N2:highway | highway | bus_stop   |
+            | N2:railway | railway | stop       |
+        When indexing
+        Then placex contains exactly
+            | object     | type       | name        |
+            | N1:amenity | restaurant | -           |
+            | N2:highway | bus_stop   | 'name': 'X' |
+            | N2:railway | stop       | 'name': 'X' |
+
+
+    Scenario: Main tag modified
+        When loading osm data
+            """
+            n10 Thighway=footway,name=X
+            n11 Tamenity=atm
+            """
+        Then place contains exactly
+            | object | class   | type    |
+            | N10    | highway | footway |
+            | N11    | amenity | atm     |
+
+        When updating osm data
+            """
+            n10 Thighway=path,name=X
+            n11 Thighway=primary
+            """
+        Then place contains exactly
+            | object | class   | type    |
+            | N10    | highway | path    |
+            | N11    | highway | primary |
+        When indexing
+        Then placex contains exactly
+            | object      | type       | name        |
+            | N10:highway | path       | 'name': 'X' |
+            | N11:highway | primary    | -           |
+
+
+    Scenario: Main tags with name, name added
+        When loading osm data
+            """
+            n45 Tlanduse=cemetry
+            n46 Tbuilding=yes
+            """
+        Then place contains exactly
+            | object | class   | type    |
+
+        When updating osm data
+            """
+            n45 Tlanduse=cemetry,name=TODO
+            n46 Tbuilding=yes,addr:housenumber=1
+            """
+        Then place contains exactly
+            | object | class   | type    |
+            | N45    | landuse | cemetry |
+            | N46    | building| yes     |
+        When indexing
+        Then placex contains exactly
+            | object      | type       | name           | address            |
+            | N45:landuse | cemetry    | 'name': 'TODO' | -                  |
+            | N46:building| yes        | -              | 'housenumber': '1' |
+
+
+    Scenario: Main tags with name, name removed
+        When loading osm data
+            """
+            n45 Tlanduse=cemetry,name=TODO
+            n46 Tbuilding=yes,addr:housenumber=1
+            """
+        Then place contains exactly
+            | object | class   | type    |
+            | N45    | landuse | cemetry |
+            | N46    | building| yes     |
+
+        When updating osm data
+            """
+            n45 Tlanduse=cemetry
+            n46 Tbuilding=yes
+            """
+        Then place contains exactly
+            | object | class   | type    |
+        When indexing
+        Then placex contains exactly
+            | object      |
+
+    Scenario: Main tags with name, name modified
+        When loading osm data
+            """
+            n45 Tlanduse=cemetry,name=TODO
+            n46 Tbuilding=yes,addr:housenumber=1
+            """
+        Then place contains exactly
+            | object | class   | type    | name            | address           |
+            | N45    | landuse | cemetry | 'name' : 'TODO' | -                 |
+            | N46    | building| yes     | -               | 'housenumber': '1'|
+
+        When updating osm data
+            """
+            n45 Tlanduse=cemetry,name=DONE
+            n46 Tbuilding=yes,addr:housenumber=10
+            """
+        Then place contains exactly
+            | object | class   | type    | name            | address            |
+            | N45    | landuse | cemetry | 'name' : 'DONE' | -                  |
+            | N46    | building| yes     | -               | 'housenumber': '10'|
+        When indexing
+        Then placex contains exactly
+            | object | class   | type    | name            | address            |
+            | N45    | landuse | cemetry | 'name' : 'DONE' | -                  |
+            | N46    | building| yes     | -               | 'housenumber': '10'|
+
+
+    Scenario: Main tag added to address only node
+        When loading osm data
+            """
+            n1 Taddr:housenumber=345
+            """
+        Then place contains exactly
+            | object | class | type  | address |
+            | N1     | place | house | 'housenumber': '345'|
+
+        When updating osm data
+            """
+            n1 Taddr:housenumber=345,building=yes
+            """
+        Then place contains exactly
+            | object | class    | type  | address |
+            | N1     | building | yes   | 'housenumber': '345'|
+        When indexing
+        Then placex contains exactly
+            | object | class    | type  | address |
+            | N1     | building | yes   | 'housenumber': '345'|
+
+
+    Scenario: Main tag removed from address only node
+        When loading osm data
+            """
+            n1 Taddr:housenumber=345,building=yes
+            """
+        Then place contains exactly
+            | object | class    | type  | address |
+            | N1     | building | yes   | 'housenumber': '345'|
+
+        When updating osm data
+            """
+            n1 Taddr:housenumber=345
+            """
+        Then place contains exactly
+            | object | class | type  | address |
+            | N1     | place | house | 'housenumber': '345'|
+        When indexing
+        Then placex contains exactly
+            | object | class | type  | address |
+            | N1     | place | house | 'housenumber': '345'|
+
+
+    Scenario: Main tags with name key, adding key name
+        When loading osm data
+            """
+            n2 Tbridge=yes
+            """
+        Then place contains exactly
+            | object | class    | type  |
+
+        When updating osm data
+            """
+            n2 Tbridge=yes,bridge:name=high
+            """
+        Then place contains exactly
+            | object | class    | type  | name           |
+            | N2     | bridge   | yes   | 'name': 'high' |
+        When indexing
+        Then placex contains exactly
+            | object | class    | type  | name           |
+            | N2     | bridge   | yes   | 'name': 'high' |
+
+
+    Scenario: Main tags with name key, deleting key name
+        When loading osm data
+            """
+            n2 Tbridge=yes,bridge:name=high
+            """
+        Then place contains exactly
+            | object | class    | type  | name           |
+            | N2     | bridge   | yes   | 'name': 'high' |
+
+        When updating osm data
+            """
+            n2 Tbridge=yes
+            """
+        Then place contains exactly
+            | object |
+        When indexing
+        Then placex contains exactly
+            | object |
+
+
+    Scenario: Main tags with name key, changing key name
+        When loading osm data
+            """
+            n2 Tbridge=yes,bridge:name=high
+            """
+        Then place contains exactly
+            | object | class    | type  | name           |
+            | N2     | bridge   | yes   | 'name': 'high' |
+
+        When updating osm data
+            """
+            n2 Tbridge=yes,bridge:name:en=high
+            """
+        Then place contains exactly
+            | object | class  | type | name              |
+            | N2     | bridge | yes  | 'name:en': 'high' |
+        When indexing
+        Then placex contains exactly
+            | object | class  | type | name              |
+            | N2     | bridge | yes  | 'name:en': 'high' |
+
+
+    Scenario: Downgrading a highway to one that is dropped without name
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=residential Nn100,n101
+          """
+        Then place contains exactly
+          | object     |
+          | W1:highway |
+
+        When updating osm data
+          """
+          w1 Thighway=service Nn100,n101
+          """
+        Then place contains exactly
+          | object     |
+        When indexing
+        Then placex contains exactly
+            | object |
+
+
+    Scenario: Upgrading a highway to one that is not dropped without name
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=service Nn100,n101
+          """
+        Then place contains exactly
+          | object     |
+
+        When updating osm data
+          """
+          w1 Thighway=unclassified Nn100,n101
+          """
+        Then place contains exactly
+          | object     |
+          | W1:highway |
+        When indexing
+        Then placex contains exactly
+          | object     |
+          | W1:highway |
+
+
+    Scenario: Downgrading a highway when a second tag is present
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=residential,tourism=hotel Nn100,n101
+          """
+        Then place contains exactly
+          | object     | type        |
+          | W1:highway | residential |
+          | W1:tourism | hotel       |
+
+        When updating osm data
+          """
+          w1 Thighway=service,tourism=hotel Nn100,n101
+          """
+        Then place contains exactly
+          | object     | type  |
+          | W1:tourism | hotel |
+        When indexing
+        Then placex contains exactly
+          | object     | type  |
+          | W1:tourism | hotel |
+
+
+    Scenario: Upgrading a highway when a second tag is present
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=service,tourism=hotel Nn100,n101
+          """
+        Then place contains exactly
+          | object     | type  |
+          | W1:tourism | hotel |
+
+        When updating osm data
+          """
+          w1 Thighway=residential,tourism=hotel Nn100,n101
+          """
+        Then place contains exactly
+          | object     | type        |
+          | W1:highway | residential |
+          | W1:tourism | hotel       |
+        When indexing
+        Then placex contains exactly
+          | object     | type        |
+          | W1:highway | residential |
+          | W1:tourism | hotel       |
+
+
+    Scenario: Replay on administrative boundary
+        When loading osm data
+          """
+          n10 x34.0 y-4.23
+          n11 x34.1 y-4.23
+          n12 x34.2 y-4.13
+          w10 Tboundary=administrative,waterway=river,name=Border,admin_level=2 Nn12,n11,n10
+          """
+        Then place contains exactly
+          | object       | type           | admin_level | name             |
+          | W10:waterway | river          | 2           | 'name': 'Border' |
+          | W10:boundary | administrative | 2           | 'name': 'Border' |
+
+        When updating osm data
+          """
+          w10 Tboundary=administrative,waterway=river,name=Border,admin_level=2 Nn12,n11,n10
+          """
+        Then place contains exactly
+          | object       | type           | admin_level | name             |
+          | W10:waterway | river          | 2           | 'name': 'Border' |
+          | W10:boundary | administrative | 2           | 'name': 'Border' |
+        When indexing
+        Then placex contains exactly
+          | object       | type           | admin_level | name             |
+          | W10:waterway | river          | 2           | 'name': 'Border' |
+
+
+    Scenario: Change admin_level on administrative boundary
+        Given the grid
+          | 10 | 11 |
+          | 13 | 12 |
+        When loading osm data
+          """
+          n10
+          n11
+          n12
+          n13
+          w10 Nn10,n11,n12,n13,n10
+          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=2 Mw10@
+          """
+        Then place contains exactly
+          | object       | admin_level |
+          | R10:boundary | 2           |
+
+        When updating osm data
+          """
+          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
+          """
+        Then place contains exactly
+          | object       | type           | admin_level |
+          | R10:boundary | administrative | 4           |
+        When indexing
+        Then placex contains exactly
+          | object       | type           | admin_level |
+          | R10:boundary | administrative | 4           |
+
+
+    Scenario: Change boundary to administrative
+        Given the grid
+          | 10 | 11 |
+          | 13 | 12 |
+        When loading osm data
+          """
+          n10
+          n11
+          n12
+          n13
+          w10 Nn10,n11,n12,n13,n10
+          r10 Ttype=multipolygon,boundary=informal,name=Border,admin_level=4 Mw10@
+          """
+        Then place contains exactly
+          | object       | type     | admin_level |
+          | R10:boundary | informal | 4           |
+
+        When updating osm data
+          """
+          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
+          """
+        Then place contains exactly
+          | object       | type           | admin_level |
+          | R10:boundary | administrative | 4           |
+        When indexing
+        Then placex contains exactly
+          | object       | type           | admin_level |
+          | R10:boundary | administrative | 4           |
+
+
+    Scenario: Change boundary away from administrative
+        Given the grid
+          | 10 | 11 |
+          | 13 | 12 |
+        When loading osm data
+          """
+          n10
+          n11
+          n12
+          n13
+          w10 Nn10,n11,n12,n13,n10
+          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
+          """
+        Then place contains exactly
+          | object       | type           | admin_level |
+          | R10:boundary | administrative | 4           |
+
+        When updating osm data
+          """
+          r10 Ttype=multipolygon,boundary=informal,name=Border,admin_level=4 Mw10@
+          """
+        Then place contains exactly
+          | object       | type     | admin_level |
+          | R10:boundary | informal | 4           |
+        When indexing
+        Then placex contains exactly
+          | object       | type     | admin_level |
+          | R10:boundary | informal | 4           |

test/bdd/steps/nominatim_environment.py

@@ -36,6 +36,7 @@ class NominatimEnvironment:
         self.api_test_db = config['API_TEST_DB']
         self.api_test_file = config['API_TEST_FILE']
         self.tokenizer = config['TOKENIZER']
+        self.import_style = config['STYLE']
         self.server_module_path = config['SERVER_MODULE_PATH']
         self.reuse_template = not config['REMOVE_TEMPLATE']
         self.keep_scenario_db = config['KEEP_TEST_DB']
@@ -107,6 +108,8 @@ class NominatimEnvironment:
         self.test_env['NOMINATIM_NOMINATIM_TOOL'] = str((self.build_dir / 'nominatim').resolve())
         if self.tokenizer is not None:
             self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer
+        if self.import_style is not None:
+            self.test_env['NOMINATIM_IMPORT_STYLE'] = self.import_style
 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path

test/bdd/steps/place_inserter.py

@@ -92,6 +92,12 @@ class PlaceColumn:
         else:
             self.columns[column] = {key: value}
 
+    def db_delete(self, cursor):
+        """ Issue a delete for the given OSM object.
+        """
+        cursor.execute('DELETE FROM place WHERE osm_type = %s and osm_id = %s',
+                       (self.columns['osm_type'] , self.columns['osm_id']))
+
     def db_insert(self, cursor):
         """ Insert the collected data into the database.
         """

test/bdd/steps/steps_db_ops.py

@@ -118,7 +118,10 @@ def update_place_table(context):
     context.nominatim.run_nominatim('refresh', '--functions')
     with context.db.cursor() as cur:
         for row in context.table:
-            PlaceColumn(context).add_row(row, False).db_insert(cur)
+            col = PlaceColumn(context).add_row(row, False)
+            col.db_delete(cur)
+            col.db_insert(cur)
+        cur.execute('SELECT flush_deleted_places()')
 
     context.nominatim.reindex_placex(context.db)
     check_database_integrity(context)
@@ -143,8 +146,10 @@ def delete_places(context, oids):
     """
     context.nominatim.run_nominatim('refresh', '--functions')
     with context.db.cursor() as cur:
+        cur.execute('TRUNCATE place_to_be_deleted')
         for oid in oids.split(','):
             NominatimID(oid).query_osm_id(cur, 'DELETE FROM place WHERE {}')
+        cur.execute('SELECT flush_deleted_places()')
 
     context.nominatim.reindex_placex(context.db)
 
@@ -185,7 +190,10 @@ def check_place_contents(context, table, exact):
 
         if exact:
             cur.execute('SELECT osm_type, osm_id, class from {}'.format(table))
-            assert expected_content == set([(r[0], r[1], r[2]) for r in cur])
+            actual = set([(r[0], r[1], r[2]) for r in cur])
+            assert expected_content == actual, \
+                   f"Missing entries: {expected_content - actual}\n" \
+                   f"Not expected in table: {actual - expected_content}"
 
 
 @then("(?P<table>placex|place) has no entry for (?P<oid>.*)")
@@ -372,4 +380,49 @@ def check_location_property_osmline(context, oid, neg):
 
         assert not todo, f"Unmatched lines in table: {list(context.table[i] for i in todo)}"
 
+@then("location_property_osmline contains(?P<exact> exactly)?")
+def check_place_contents(context, exact):
+    """ Check contents of the interpolation table. Each row represents a table row
+        and all data must match. Data not present in the expected table, may
+        be arbitry. The rows are identified via the 'object' column which must
+        have an identifier of the form '<osm id>[:<startnumber>]'. When multiple
+        rows match (for example because 'startnumber' was left out and there are
+        multiple entries for the given OSM object) then all must match. All
+        expected rows are expected to be present with at least one database row.
+        When 'exactly' is given, there must not be additional rows in the database.
+    """
+    with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
+        expected_content = set()
+        for row in context.table:
+            if ':' in row['object']:
+                nid, start = row['object'].split(':', 2)
+                start = int(start)
+            else:
+                nid, start = row['object'], None
+
+            query = """SELECT *, ST_AsText(linegeo) as geomtxt,
+                              ST_GeometryType(linegeo) as geometrytype
+                       FROM location_property_osmline WHERE osm_id=%s"""
+
+            if ':' in row['object']:
+                query += ' and startnumber = %s'
+                params = [int(val) for val in row['object'].split(':', 2)]
+            else:
+                params = (int(row['object']), )
+
+            cur.execute(query, params)
+            assert cur.rowcount > 0, "No rows found for " + row['object']
+
+            for res in cur:
+                if exact:
+                    expected_content.add((res['osm_id'], res['startnumber']))
+
+                DBRow(nid, res, context).assert_row(row, ['object'])
+
+        if exact:
+            cur.execute('SELECT osm_id, startnumber from location_property_osmline')
+            actual = set([(r[0], r[1]) for r in cur])
+            assert expected_content == actual, \
+                   f"Missing entries: {expected_content - actual}\n" \
+                   f"Not expected in table: {actual - expected_content}"
 

test/bdd/steps/steps_osm_data.py

@@ -10,6 +10,7 @@ import os
 from pathlib import Path
 
 from nominatim.tools.exec_utils import run_osm2pgsql
+from nominatim.tools.replication import run_osm2pgsql_updates
 
 from geometry_alias import ALIASES
 
@@ -17,7 +18,8 @@ def get_osm2pgsql_options(nominatim_env, fname, append):
     return dict(import_file=fname,
                 osm2pgsql=str(nominatim_env.build_dir / 'osm2pgsql' / 'osm2pgsql'),
                 osm2pgsql_cache=50,
-                osm2pgsql_style=str(nominatim_env.src_dir / 'settings' / 'import-extratags.style'),
+                osm2pgsql_style=str(nominatim_env.get_test_config().get_import_style_file()),
+                osm2pgsql_style_path=nominatim_env.get_test_config().config_dir,
                 threads=1,
                 dsn=nominatim_env.get_libpq_dsn(),
                 flatnode_file='',
@@ -117,6 +119,15 @@ def update_from_osm_file(context):
     # create an OSM file and import it
     fname = write_opl_file(context.text, context.osm)
     try:
-        run_osm2pgsql(get_osm2pgsql_options(context.nominatim, fname, append=True))
+        run_osm2pgsql_updates(context.db,
+                              get_osm2pgsql_options(context.nominatim, fname, append=True))
     finally:
         os.remove(fname)
+
+@when('indexing')
+def index_database(context):
+    """
+    Run the Nominatim indexing step. This will process data previously
+    loaded with 'updating osm data'
+    """
+    context.nominatim.run_nominatim('index')

vagrant/Install-on-Ubuntu-18.sh

@@ -24,7 +24,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
     sudo apt install -y php-cgi
     sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
                         libboost-filesystem-dev libexpat1-dev zlib1g-dev\
-                        libbz2-dev libpq-dev \
+                        libbz2-dev libpq-dev liblua5.3-dev lua5.3\
                         postgresql-10-postgis-2.4 \
                         postgresql-contrib-10 postgresql-10-postgis-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-pip \

vagrant/Install-on-Ubuntu-20.sh

@@ -23,7 +23,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
     sudo apt install -y php-cgi
     sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
                         libboost-filesystem-dev libexpat1-dev zlib1g-dev \
-                        libbz2-dev libpq-dev \
+                        libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
                         postgresql-12-postgis-3 \
                         postgresql-contrib-12 postgresql-12-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \

vagrant/Install-on-Ubuntu-22.sh

@@ -23,7 +23,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
     sudo apt install -y php-cgi
     sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
                         libboost-filesystem-dev libexpat1-dev zlib1g-dev \
-                        libbz2-dev libpq-dev \
+                        libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
                         postgresql-server-dev-14 postgresql-14-postgis-3 \
                         postgresql-contrib-14 postgresql-14-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \