From e93c6809a93eb901745acbe99befd890b5b0ce9a Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 16 Feb 2026 14:11:58 +0100 Subject: [PATCH 01/10] import interpolation into separate place table --- lib-lua/themes/nominatim/init.lua | 40 ++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index 129a0d93..41eaafe1 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -77,7 +77,19 @@ local table_definitions = { indexes = { { column = 'postcode', method = 'btree' } } - } + }, + place_interpolation = { + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'type', type = 'text', not_null = true }, + { column = 'address', type = 'hstore' }, + { column = 'nodes', type = 'text', sql_type = 'bigint[]' }, + { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true }, + }, + indexes = { + { column = 'nodes', method = 'gin' } + } + } } local insert_row = {} @@ -703,8 +715,30 @@ function module.process_tags(o) o.address['country'] = nil end - if o.address.interpolation ~= nil then - o:write_place('place', 'houses', PlaceTransform.always) + if o.address.interpolation ~= nil + and (o.address.housenumber == nil or string.find(o.address.housenumber, '-') ~= nil) then + if o:geometry_is_valid() then + local extra_addr = nil + for k, v in pairs(o.address) do + if k ~= 'interpolation' then + if extra_addr == nil then + extra_addr = {k = v} + else + extra_addr[k] = v + end + end + end + local nodes = nil + if o.address.housenumber == nil and o.object.nodes ~= nil then + nodes = '{' .. table.concat(o.object.nodes, ',') .. '}' + end + insert_row.place_interpolation{ + type = o.address.interpolation, + address = extra_addr, + nodes = nodes, + geometry = o.geometry + } + end return end From a115eeeb40fed6eee43ddbb3b7d52009828e8145 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 16 Feb 2026 14:18:39 +0100 Subject: [PATCH 02/10] copy interpolation data from new place_interpolation table --- lib-sql/tables/interpolation.sql | 2 ++ src/nominatim_db/tools/database_import.py | 13 +++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib-sql/tables/interpolation.sql b/lib-sql/tables/interpolation.sql index 9a93f8d1..a7b53209 100644 --- a/lib-sql/tables/interpolation.sql +++ b/lib-sql/tables/interpolation.sql @@ -8,10 +8,12 @@ DROP TABLE IF EXISTS location_property_osmline; CREATE TABLE location_property_osmline ( place_id BIGINT NOT NULL, + osm_type CHAR(1) NOT NULL, osm_id BIGINT NOT NULL, parent_place_id BIGINT, geometry_sector INTEGER NOT NULL, indexed_date TIMESTAMP, + type TEXT, startnumber INTEGER, endnumber INTEGER, step SMALLINT, diff --git a/src/nominatim_db/tools/database_import.py b/src/nominatim_db/tools/database_import.py index 9af74bf6..388cca0f 100644 --- a/src/nominatim_db/tools/database_import.py +++ b/src/nominatim_db/tools/database_import.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Functions for setting up and importing a new Nominatim database. @@ -219,19 +219,16 @@ async def load_data(dsn: str, threads: int) -> None: pysql.SQL("""INSERT INTO placex ({columns}) SELECT {columns} FROM place WHERE osm_id % {total} = {mod} - AND NOT (class='place' - and (type='houses' or type='postcode')) - AND ST_IsValid(geometry) """).format(columns=_COPY_COLUMNS, total=pysql.Literal(placex_threads), mod=pysql.Literal(imod)), None) # Interpolations need to be copied separately await pool.put_query(""" - INSERT INTO location_property_osmline (osm_id, address, linegeo) - SELECT osm_id, address, geometry FROM place - WHERE class='place' and type='houses' and osm_type='W' - and ST_GeometryType(geometry) = 'ST_LineString' """, None) + INSERT INTO location_property_osmline (osm_type, osm_id, type, address, linegeo) + SELECT osm_type, osm_id, type, address, geometry + FROM place_interpolation + """, None) progress.cancel() From c2d6821f2fb71dc4d3cb2d80dd373990378f6621 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 16 Feb 2026 16:25:19 +0100 Subject: [PATCH 03/10] adapt interpolation handling to use separate place_interpolation table --- lib-sql/functions/interpolation.sql | 443 +++++++++++++++----------- lib-sql/functions/place_triggers.sql | 59 ---- lib-sql/functions/placex_triggers.sql | 10 +- lib-sql/functions/utils.sql | 24 +- lib-sql/indices.sql | 17 +- lib-sql/table-triggers.sql | 7 +- lib-sql/tables.sql | 5 - 7 files changed, 296 insertions(+), 269 deletions(-) diff --git a/lib-sql/functions/interpolation.sql b/lib-sql/functions/interpolation.sql index 7e8877cc..a31c1793 100644 --- a/lib-sql/functions/interpolation.sql +++ b/lib-sql/functions/interpolation.sql @@ -2,11 +2,106 @@ -- -- This file is part of Nominatim. (https://nominatim.org) -- --- Copyright (C) 2022 by the Nominatim developer community. +-- Copyright (C) 2026 by the Nominatim developer community. -- For a full list of authors see the git log. -- Functions for address interpolation objects in location_property_osmline. +CREATE OR REPLACE FUNCTION place_interpolation_insert() + RETURNS TRIGGER + AS $$ +DECLARE + existing RECORD; + existingplacex BIGINT[]; + +BEGIN + -- Remove the place from the list of places to be deleted + DELETE FROM place_interpolation_to_be_deleted pdel + WHERE pdel.osm_type = NEW.osm_type and pdel.osm_id = NEW.osm_id; + + SELECT * INTO existing FROM place_interpolation p + WHERE p.osm_type = NEW.osm_type AND p.osm_id = NEW.osm_id; + + IF NOT (NEW.type in ('odd', 'even', 'all') OR NEW.type similar to '[1-9]') THEN + -- the new interpolation is illegal, simply remove existing entries + DELETE FROM location_property_osmline o + WHERE o.osm_type = NEW.osm_type AND o.osm_id = NEW.osm_id; + ELSE + -- Get the existing entry from the interpolation table. + SELECT array_agg(place_id) INTO existingplacex + FROM location_property_osmline o + WHERE o.osm_type = NEW.osm_type AND o.osm_id = NEW.osm_id; + + IF array_length(existingplacex, 1) is NULL THEN + INSERT INTO location_property_osmline (osm_type, osm_id, type, address, linegeo) + VALUES (NEW.osm_type, NEW.osm_id, NEW.type, NEW.address, NEW.geometry); + ELSE + -- Update the interpolation table: + -- The first entry gets the original data, all other entries + -- are removed and will be recreated on indexing. + -- (An interpolation can be split up, if it has more than 2 address nodes) + -- Update unconditionally here as the changes might be coming from the + -- nodes on the interpolation. + UPDATE location_property_osmline + SET type = NEW.type, + address = NEW.address, + linegeo = NEW.geometry, + startnumber = null, + indexed_status = 1 + WHERE place_id = existingplacex[1]; + IF array_length(existingplacex, 1) > 1 THEN + DELETE FROM location_property_osmline + WHERE place_id = any(existingplacex[2:]); + END IF; + END IF; + + -- need to invalidate nodes because they might copy address info + IF NEW.address is not NULL + AND (existing.osm_type is NULL + OR coalesce(existing.address, ''::hstore) != NEW.address) + THEN + UPDATE placex SET indexed_status = 2 + WHERE osm_type = 'N' AND osm_id = ANY(NEW.nodes) + AND indexed_status = 0; + END IF; + END IF; + + -- finally update/insert place_interpolation itself + + IF existing.osm_type is not NULL THEN + IF existing.type != NEW.type + OR coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore) + OR existing.geometry::text != NEW.geometry::text + THEN + UPDATE place_interpolation p + SET type = NEW.type, + address = NEW.address, + geometry = NEW.geometry + WHERE p.osm_type = NEW.osm_type AND p.osm_id = NEW.osm_id; + END IF; + + RETURN NULL; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION place_interpolation_delete() + RETURNS TRIGGER + AS $$ +DECLARE + deferred BOOLEAN; +BEGIN + {% if debug %}RAISE WARNING 'Delete for % % %/%', OLD.osm_type, OLD.osm_id, OLD.class, OLD.type;{% endif %} + + INSERT INTO place_interpolation_to_be_deleted (osm_type, osm_id) + VALUES(OLD.osm_type, OLD.osm_id); + + RETURN NULL; +END; +$$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION get_interpolation_address(in_address HSTORE, wayid BIGINT) RETURNS HSTORE @@ -28,7 +123,7 @@ BEGIN and indexed_status < 100 LOOP -- mark it as a derived address - RETURN location.address || in_address || hstore('_inherited', ''); + RETURN location.address || coalesce(in_address, '{}'::hstore) || hstore('_inherited', ''); END LOOP; RETURN in_address; @@ -73,51 +168,6 @@ $$ LANGUAGE plpgsql STABLE PARALLEL SAFE; -CREATE OR REPLACE FUNCTION reinsert_interpolation(way_id BIGINT, addr HSTORE, - geom GEOMETRY) - RETURNS INT - AS $$ -DECLARE - existing BIGINT[]; -BEGIN - IF addr is NULL OR NOT addr ? 'interpolation' - OR NOT (addr->'interpolation' in ('odd', 'even', 'all') - or addr->'interpolation' similar to '[1-9]') - THEN - -- the new interpolation is illegal, simply remove existing entries - DELETE FROM location_property_osmline WHERE osm_id = way_id; - ELSE - -- Get the existing entry from the interpolation table. - SELECT array_agg(place_id) INTO existing - FROM location_property_osmline WHERE osm_id = way_id; - - IF existing IS NULL or array_length(existing, 1) = 0 THEN - INSERT INTO location_property_osmline (osm_id, address, linegeo) - VALUES (way_id, addr, geom); - ELSE - -- Update the interpolation table: - -- The first entry gets the original data, all other entries - -- are removed and will be recreated on indexing. - -- (An interpolation can be split up, if it has more than 2 address nodes) - UPDATE location_property_osmline - SET address = addr, - linegeo = geom, - startnumber = null, - indexed_status = 1 - WHERE place_id = existing[1]; - IF array_length(existing, 1) > 1 THEN - DELETE FROM location_property_osmline - WHERE place_id = any(existing[2:]); - END IF; - END IF; - END IF; - - RETURN 1; -END; -$$ -LANGUAGE plpgsql; - - CREATE OR REPLACE FUNCTION osmline_insert() RETURNS TRIGGER AS $$ @@ -128,20 +178,32 @@ BEGIN NEW.indexed_date := now(); IF NEW.indexed_status IS NULL THEN - IF NEW.address is NULL OR NOT NEW.address ? 'interpolation' - OR NOT (NEW.address->'interpolation' in ('odd', 'even', 'all') - or NEW.address->'interpolation' similar to '[1-9]') - THEN - -- alphabetic interpolation is not supported + IF NOT(NEW.type in ('odd', 'even', 'all') OR NEW.type similar to '[1-9]') THEN + -- alphabetic interpolation is not supported + RETURN NULL; + END IF; + + IF NEW.address is not NULL AND NEW.address ? 'housenumber' THEN + IF NEW.address->'housenumber' not similar to '[0-9]+-[0-9]+' THEN + -- housenumber needs to look like an interpolation RETURN NULL; END IF; - NEW.indexed_status := 1; --STATUS_NEW + centroid := ST_Centroid(NEW.linegeo); + -- interpolation of a housenumber, make sure we have a line to interpolate on + NEW.geometry := ST_MakeLine(centroid, ST_Project(centroid, 0.0000001, 0)); + ELSE centroid := get_center_point(NEW.linegeo); - NEW.country_code := lower(get_country_code(centroid)); + IF NEW.osm_type != 'W' THEN + RETURN NULL; + END IF; + END IF; - NEW.partition := get_partition(NEW.country_code); - NEW.geometry_sector := geometry_sector(NEW.partition, centroid); + NEW.indexed_status := 1; --STATUS_NEW + NEW.country_code := lower(get_country_code(centroid)); + + NEW.partition := get_partition(NEW.country_code); + NEW.geometry_sector := geometry_sector(NEW.partition, centroid); END IF; RETURN NEW; @@ -167,6 +229,7 @@ DECLARE sectiongeo GEOMETRY; postcode TEXT; stepmod SMALLINT; + splitstring TEXT[]; BEGIN -- deferred delete IF OLD.indexed_status = 100 THEN @@ -182,157 +245,173 @@ BEGIN get_center_point(NEW.linegeo), NEW.linegeo); - -- Cannot find a parent street. We will not be able to display a reliable - -- address, so drop entire interpolation. - IF NEW.parent_place_id is NULL THEN - DELETE FROM location_property_osmline where place_id = OLD.place_id; - RETURN NULL; - END IF; - NEW.token_info := token_strip_info(NEW.token_info); IF NEW.address ? '_inherited' THEN - NEW.address := hstore('interpolation', NEW.address->'interpolation'); + NEW.address := NULL; END IF; -- If the line was newly inserted, split the line as necessary. - IF OLD.indexed_status = 1 THEN - IF NEW.address->'interpolation' in ('odd', 'even') THEN + IF NEW.parent_place_id is not NULL AND NEW.startnumber is NULL THEN + IF NEW.type in ('odd', 'even') THEN NEW.step := 2; - stepmod := CASE WHEN NEW.address->'interpolation' = 'odd' THEN 1 ELSE 0 END; + stepmod := CASE WHEN NEW.type = 'odd' THEN 1 ELSE 0 END; ELSE - NEW.step := CASE WHEN NEW.address->'interpolation' = 'all' - THEN 1 - ELSE (NEW.address->'interpolation')::SMALLINT END; + NEW.step := CASE WHEN NEW.type = 'all' THEN 1 ELSE (NEW.type)::SMALLINT END; stepmod := NULL; END IF; - SELECT nodes INTO waynodes - FROM planet_osm_ways WHERE id = NEW.osm_id; + IF NEW.address is not NULL AND NEW.address ? 'housenumer' THEN + -- interpolation interval is in housenumber + splitstring := string_to_array(NEW.address->'housenumber', '-'); + NEW.startnumber := (splitstring[1])::INTEGER; + NEW.endnumber := (splitstring[2])::INTEGER; - IF array_upper(waynodes, 1) IS NULL THEN - RETURN NEW; - END IF; - - linegeo := null; - SELECT null::integer as hnr INTO prevnode; - - -- Go through all nodes on the interpolation line that have a housenumber. - FOR nextnode IN - SELECT DISTINCT ON (nodeidpos) - osm_id, address, geometry, - -- Take the postcode from the node only if it has a housenumber itself. - -- Note that there is a corner-case where the node has a wrongly - -- formatted postcode and therefore 'postcode' contains a derived - -- variant. - CASE WHEN address ? 'postcode' THEN placex.postcode ELSE NULL::text END as postcode, - (address->'housenumber')::integer as hnr - FROM placex, generate_series(1, array_upper(waynodes, 1)) nodeidpos - WHERE osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT - and address is not NULL and address ? 'housenumber' - and address->'housenumber' ~ '^[0-9]{1,6}$' - and ST_Distance(NEW.linegeo, geometry) < 0.0005 - ORDER BY nodeidpos - LOOP - {% if debug %}RAISE WARNING 'processing point % (%)', nextnode.hnr, ST_AsText(nextnode.geometry);{% endif %} - IF linegeo is null THEN - linegeo := NEW.linegeo; - ELSE - splitpoint := ST_LineLocatePoint(linegeo, nextnode.geometry); - IF splitpoint = 0 THEN - -- Corner case where the splitpoint falls on the first point - -- and thus would not return a geometry. Skip that section. - sectiongeo := NULL; - ELSEIF splitpoint = 1 THEN - -- Point is at the end of the line. - sectiongeo := linegeo; - linegeo := NULL; - ELSE - -- Split the line. - sectiongeo := ST_LineSubstring(linegeo, 0, splitpoint); - linegeo := ST_LineSubstring(linegeo, splitpoint, 1); + IF stepmod is not NULL THEN + IF NEW.startnumber % NEW.step != stepmod THEN + NEW.startnumber := NEW.startnumber + 1; END IF; + IF NEW.endnumber % NEW.step != stepmod THEN + NEW.endnumber := NEW.endnumber - 1; + END IF; + ELSE + NEW.endnumber := NEW.startnumber + ((NEW.endnumber - NEW.startnumber) / NEW.step) * NEW.step; END IF; - IF prevnode.hnr is not null - -- Check if there are housenumbers to interpolate between the - -- regularly mapped housenumbers. - -- (Conveniently also fails if one of the house numbers is not a number.) - and abs(prevnode.hnr - nextnode.hnr) > NEW.step - -- If the interpolation geometry is broken or two nodes are at the - -- same place, then splitting might produce a point. Ignore that. - and ST_GeometryType(sectiongeo) = 'ST_LineString' - THEN - IF prevnode.hnr < nextnode.hnr THEN - startnumber := prevnode.hnr; - endnumber := nextnode.hnr; - ELSE - startnumber := nextnode.hnr; - endnumber := prevnode.hnr; - sectiongeo := ST_Reverse(sectiongeo); - END IF; + IF NEW.startnumber = NEW.endnumber THEN + NEW.geometry := ST_PointN(NEW.geometry, 1); + ELSEIF NEW.startnumber > NEW.endnumber THEN + NEW.startnumber := NULL; + END IF; + ELSE + -- classic interpolation way + SELECT nodes INTO waynodes + FROM planet_osm_ways WHERE id = NEW.osm_id; - -- Adjust the interpolation, so that only inner housenumbers - -- are taken into account. - IF stepmod is null THEN - newstart := startnumber + NEW.step; + IF array_upper(waynodes, 1) IS NULL THEN + RETURN NEW; + END IF; + + linegeo := null; + SELECT null::integer as hnr INTO prevnode; + + -- Go through all nodes on the interpolation line that have a housenumber. + FOR nextnode IN + SELECT DISTINCT ON (nodeidpos) + osm_id, address, geometry, + -- Take the postcode from the node only if it has a housenumber itself. + -- Note that there is a corner-case where the node has a wrongly + -- formatted postcode and therefore 'postcode' contains a derived + -- variant. + CASE WHEN address ? 'postcode' THEN placex.postcode ELSE NULL::text END as postcode, + (address->'housenumber')::integer as hnr + FROM placex, generate_series(1, array_upper(waynodes, 1)) nodeidpos + WHERE osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT + and address is not NULL and address ? 'housenumber' + and address->'housenumber' ~ '^[0-9]{1,6}$' + and ST_Distance(NEW.linegeo, geometry) < 0.0005 + ORDER BY nodeidpos + LOOP + {% if debug %}RAISE WARNING 'processing point % (%)', nextnode.hnr, ST_AsText(nextnode.geometry);{% endif %} + IF linegeo is null THEN + linegeo := NEW.linegeo; ELSE - newstart := startnumber + 1; - moddiff := newstart % NEW.step - stepmod; - IF moddiff < 0 THEN - newstart := newstart + (NEW.step + moddiff); + splitpoint := ST_LineLocatePoint(linegeo, nextnode.geometry); + IF splitpoint = 0 THEN + -- Corner case where the splitpoint falls on the first point + -- and thus would not return a geometry. Skip that section. + sectiongeo := NULL; + ELSEIF splitpoint = 1 THEN + -- Point is at the end of the line. + sectiongeo := linegeo; + linegeo := NULL; ELSE - newstart := newstart + moddiff; + -- Split the line. + sectiongeo := ST_LineSubstring(linegeo, 0, splitpoint); + linegeo := ST_LineSubstring(linegeo, splitpoint, 1); END IF; END IF; - newend := newstart + ((endnumber - 1 - newstart) / NEW.step) * NEW.step; - -- If newstart and newend are the same, then this returns a point. - sectiongeo := ST_LineSubstring(sectiongeo, - (newstart - startnumber)::float / (endnumber - startnumber)::float, - (newend - startnumber)::float / (endnumber - startnumber)::float); - startnumber := newstart; - endnumber := newend; + IF prevnode.hnr is not null + -- Check if there are housenumbers to interpolate between the + -- regularly mapped housenumbers. + -- (Conveniently also fails if one of the house numbers is not a number.) + and abs(prevnode.hnr - nextnode.hnr) > NEW.step + -- If the interpolation geometry is broken or two nodes are at the + -- same place, then splitting might produce a point. Ignore that. + and ST_GeometryType(sectiongeo) = 'ST_LineString' + THEN + IF prevnode.hnr < nextnode.hnr THEN + startnumber := prevnode.hnr; + endnumber := nextnode.hnr; + ELSE + startnumber := nextnode.hnr; + endnumber := prevnode.hnr; + sectiongeo := ST_Reverse(sectiongeo); + END IF; - -- determine postcode - postcode := coalesce(prevnode.postcode, nextnode.postcode, postcode); - IF postcode is NULL and NEW.parent_place_id > 0 THEN - SELECT placex.postcode FROM placex - WHERE place_id = NEW.parent_place_id INTO postcode; - END IF; - IF postcode is NULL THEN - postcode := get_nearest_postcode(NEW.country_code, nextnode.geometry); + -- Adjust the interpolation, so that only inner housenumbers + -- are taken into account. + IF stepmod is null THEN + newstart := startnumber + NEW.step; + ELSE + newstart := startnumber + 1; + moddiff := newstart % NEW.step - stepmod; + IF moddiff < 0 THEN + newstart := newstart + (NEW.step + moddiff); + ELSE + newstart := newstart + moddiff; + END IF; + END IF; + newend := newstart + ((endnumber - 1 - newstart) / NEW.step) * NEW.step; + + -- If newstart and newend are the same, then this returns a point. + sectiongeo := ST_LineSubstring(sectiongeo, + (newstart - startnumber)::float / (endnumber - startnumber)::float, + (newend - startnumber)::float / (endnumber - startnumber)::float); + startnumber := newstart; + endnumber := newend; + + -- determine postcode + postcode := coalesce(prevnode.postcode, nextnode.postcode, postcode); + IF postcode is NULL and NEW.parent_place_id > 0 THEN + SELECT placex.postcode FROM placex + WHERE place_id = NEW.parent_place_id INTO postcode; + END IF; + IF postcode is NULL THEN + postcode := get_nearest_postcode(NEW.country_code, nextnode.geometry); + END IF; + + -- Add the interpolation. If this is the first segment, just modify + -- the interpolation to be inserted, otherwise add an additional one + -- (marking it indexed already). + IF NEW.startnumber IS NULL THEN + NEW.startnumber := startnumber; + NEW.endnumber := endnumber; + NEW.linegeo := ST_ReducePrecision(sectiongeo, 0.0000001); + NEW.postcode := postcode; + ELSE + INSERT INTO location_property_osmline + (linegeo, partition, osm_type, osm_id, parent_place_id, + startnumber, endnumber, step, type, + address, postcode, country_code, + geometry_sector, indexed_status) + VALUES (ST_ReducePrecision(sectiongeo, 0.0000001), + NEW.partition, NEW.osm_type, NEW.osm_id, NEW.parent_place_id, + startnumber, endnumber, NEW.step, NEW.type, + NEW.address, postcode, + NEW.country_code, NEW.geometry_sector, 0); + END IF; END IF; - -- Add the interpolation. If this is the first segment, just modify - -- the interpolation to be inserted, otherwise add an additional one - -- (marking it indexed already). - IF NEW.startnumber IS NULL THEN - NEW.startnumber := startnumber; - NEW.endnumber := endnumber; - NEW.linegeo := ST_ReducePrecision(sectiongeo, 0.0000001); - NEW.postcode := postcode; - ELSE - INSERT INTO location_property_osmline - (linegeo, partition, osm_id, parent_place_id, - startnumber, endnumber, step, - address, postcode, country_code, - geometry_sector, indexed_status) - VALUES (ST_ReducePrecision(sectiongeo, 0.0000001), - NEW.partition, NEW.osm_id, NEW.parent_place_id, - startnumber, endnumber, NEW.step, - NEW.address, postcode, - NEW.country_code, NEW.geometry_sector, 0); + -- early break if we are out of line string, + -- might happen when a line string loops back on itself + IF linegeo is null or ST_GeometryType(linegeo) != 'ST_LineString' THEN + RETURN NEW; END IF; - END IF; - -- early break if we are out of line string, - -- might happen when a line string loops back on itself - IF linegeo is null or ST_GeometryType(linegeo) != 'ST_LineString' THEN - RETURN NEW; - END IF; - - prevnode := nextnode; - END LOOP; + prevnode := nextnode; + END LOOP; + END IF; END IF; RETURN NEW; diff --git a/lib-sql/functions/place_triggers.sql b/lib-sql/functions/place_triggers.sql index e102128a..7ebae7d5 100644 --- a/lib-sql/functions/place_triggers.sql +++ b/lib-sql/functions/place_triggers.sql @@ -14,7 +14,6 @@ DECLARE existing RECORD; existingplacex RECORD; existingline BIGINT[]; - interpol RECORD; BEGIN {% if debug %} RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry); @@ -55,41 +54,6 @@ BEGIN DELETE from import_polygon_error where osm_type = NEW.osm_type and osm_id = NEW.osm_id; DELETE from import_polygon_delete where osm_type = NEW.osm_type and osm_id = NEW.osm_id; - -- ---- Interpolation Lines - - IF NEW.class='place' and NEW.type='houses' - and NEW.osm_type='W' and ST_GeometryType(NEW.geometry) = 'ST_LineString' - THEN - PERFORM reinsert_interpolation(NEW.osm_id, NEW.address, NEW.geometry); - - -- Now invalidate all address nodes on the line. - -- They get their parent from the interpolation. - UPDATE placex p SET indexed_status = 2 - FROM planet_osm_ways w - WHERE w.id = NEW.osm_id and p.osm_type = 'N' and p.osm_id = any(w.nodes) - and indexed_status = 0; - - -- If there is already an entry in place, just update that, if necessary. - IF existing.osm_type is not null THEN - IF coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore) - OR existing.geometry::text != NEW.geometry::text - THEN - UPDATE place - SET name = NEW.name, - address = NEW.address, - extratags = NEW.extratags, - admin_level = NEW.admin_level, - geometry = NEW.geometry - WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id - and class = NEW.class and type = NEW.type; - END IF; - - RETURN NULL; - END IF; - - RETURN NEW; - END IF; - -- ---- All other place types. -- When an area is changed from large to small: log and discard change @@ -109,29 +73,6 @@ BEGIN RETURN null; END IF; - -- If an address node is part of a interpolation line and changes or is - -- newly inserted (happens when the node already existed but now gets address - -- information), then mark the interpolation line for reparenting. - -- (Already here, because interpolation lines are reindexed before nodes, - -- so in the second call it would be too late.) - IF NEW.osm_type='N' - and coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore) - THEN - FOR interpol IN - SELECT DISTINCT osm_id, address, geometry FROM place, planet_osm_ways w - WHERE NEW.geometry && place.geometry - and place.osm_type = 'W' - and place.address ? 'interpolation' - and exists (SELECT * FROM location_property_osmline - WHERE osm_id = place.osm_id - and indexed_status in (0, 2)) - and w.id = place.osm_id and NEW.osm_id = any (w.nodes) - LOOP - PERFORM reinsert_interpolation(interpol.osm_id, interpol.address, - interpol.geometry); - END LOOP; - END IF; - -- Get the existing placex entry. SELECT * INTO existingplacex FROM placex diff --git a/lib-sql/functions/placex_triggers.sql b/lib-sql/functions/placex_triggers.sql index abc4aae9..00868589 100644 --- a/lib-sql/functions/placex_triggers.sql +++ b/lib-sql/functions/placex_triggers.sql @@ -53,12 +53,10 @@ BEGIN -- See if we can inherit additional address tags from an interpolation. -- These will become permanent. FOR location IN - SELECT (address - 'interpolation'::text - 'housenumber'::text) as address - FROM place, planet_osm_ways w - WHERE place.osm_type = 'W' and place.address ? 'interpolation' - and place.geometry && p.geometry - and place.osm_id = w.id - and p.osm_id = any(w.nodes) + SELECT address as address + FROM place_interpolation + WHERE p.osm_id = any(place_interpolation.nodes) + AND address is not NULL AND not address ? 'housenumber' LOOP result.address := location.address || result.address; END LOOP; diff --git a/lib-sql/functions/utils.sql b/lib-sql/functions/utils.sql index bc2dfdca..878301a0 100644 --- a/lib-sql/functions/utils.sql +++ b/lib-sql/functions/utils.sql @@ -624,18 +624,22 @@ BEGIN and placex.type = place_to_be_deleted.type and not deferred; - -- Mark for delete in interpolations - UPDATE location_property_osmline SET indexed_status = 100 FROM place_to_be_deleted - WHERE place_to_be_deleted.osm_type = 'W' - and place_to_be_deleted.class = 'place' - and place_to_be_deleted.type = 'houses' - and location_property_osmline.osm_id = place_to_be_deleted.osm_id - and not deferred; + -- Clear todo list. + TRUNCATE TABLE place_to_be_deleted; - -- Clear todo list. - TRUNCATE TABLE place_to_be_deleted; + -- delete from place_interpolation table + ALTER TABLE place_interpolation DISABLE TRIGGER place_interpolation_before_delete; + DELETE FROM place_interpolation p USING place_interpolation_to_be_deleted d + WHERE p.osm_type = d.osm_type AND p.osm_id = d.osm_id; + ALTER TABLE place_interpolation ENABLE TRIGGER place_interpolation_before_delete; - RETURN NULL; + UPDATE location_property_osmline o SET indexed_status = 100 + FROM place_interpolation_to_be_deleted d + WHERE o.osm_type = d.osm_Type AND o.osm_id = d.osm_id; + + TRUNCATE TABLE place_interpolation_to_be_deleted; + + RETURN NULL; END; $$ LANGUAGE plpgsql; diff --git a/lib-sql/indices.sql b/lib-sql/indices.sql index c23f07ae..f3b79705 100644 --- a/lib-sql/indices.sql +++ b/lib-sql/indices.sql @@ -2,7 +2,7 @@ -- -- This file is part of Nominatim. (https://nominatim.org) -- --- Copyright (C) 2025 by the Nominatim developer community. +-- Copyright (C) 2026 by the Nominatim developer community. -- For a full list of authors see the git log. -- Indices used only during search and update. @@ -67,11 +67,16 @@ CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id --- -- Table needed for running updates with osm2pgsql on place. CREATE TABLE IF NOT EXISTS place_to_be_deleted ( - osm_type CHAR(1), - osm_id BIGINT, - class TEXT, - type TEXT, - deferred BOOLEAN + osm_type CHAR(1) NOT NULL, + osm_id BIGINT NOT NULL, + class TEXT NOT NULL, + type TEXT NOT NULL, + deferred BOOLEAN NOT NULL + ); + + CREATE TABLE IF NOT EXISTS place_interpolation_to_be_deleted ( + osm_type CHAR(1) NOT NULL, + osm_id BIGINT NOT NULL ); --- CREATE INDEX IF NOT EXISTS idx_location_postcodes_parent_place_id diff --git a/lib-sql/table-triggers.sql b/lib-sql/table-triggers.sql index f9c3f890..eba6ac0e 100644 --- a/lib-sql/table-triggers.sql +++ b/lib-sql/table-triggers.sql @@ -2,7 +2,7 @@ -- -- This file is part of Nominatim. (https://nominatim.org) -- --- Copyright (C) 2025 by the Nominatim developer community. +-- Copyright (C) 2026 by the Nominatim developer community. -- For a full list of authors see the git log. -- insert creates the location tables, creates location indexes if indexed == true @@ -31,3 +31,8 @@ CREATE TRIGGER location_postcodes_before_delete BEFORE DELETE ON location_postco FOR EACH ROW EXECUTE PROCEDURE postcodes_delete(); CREATE TRIGGER location_postcodes_before_insert BEFORE INSERT ON location_postcodes FOR EACH ROW EXECUTE PROCEDURE postcodes_insert(); + +CREATE TRIGGER place_interpolation_before_insert BEFORE INSERT ON place_interpolation + FOR EACH ROW EXECUTE PROCEDURE place_interpolation_insert(); +CREATE TRIGGER place_interpolation_before_delete BEFORE DELETE ON place_interpolation + FOR EACH ROW EXECUTE PROCEDURE place_interpolation_delete(); diff --git a/lib-sql/tables.sql b/lib-sql/tables.sql index c9ccb032..023486a5 100644 --- a/lib-sql/tables.sql +++ b/lib-sql/tables.sql @@ -32,8 +32,3 @@ CREATE INDEX planet_osm_rels_relation_members_idx ON planet_osm_rels USING gin(p WITH (fastupdate=off) {{db.tablespace.address_index}}; {% endif %} - --- Needed for lookups if a node is part of an interpolation. -CREATE INDEX IF NOT EXISTS idx_place_interpolations - ON place USING gist(geometry) {{db.tablespace.address_index}} - WHERE osm_type = 'W' and address ? 'interpolation'; From c0f1aeea4d612618403673940eddaa9968813321 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 16 Feb 2026 16:25:54 +0100 Subject: [PATCH 04/10] adapt unit tests to use separate interpolation table --- test/python/conftest.py | 45 +++++++++++++++++++---- test/python/tools/test_admin.py | 24 ++++++++---- test/python/tools/test_database_import.py | 8 ++-- 3 files changed, 58 insertions(+), 19 deletions(-) diff --git a/test/python/conftest.py b/test/python/conftest.py index cfd3a7c3..9617be5a 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -192,9 +192,9 @@ def place_table(temp_db_with_extensions, table_factory): type text NOT NULL, name hstore, admin_level smallint, - address hstore, - extratags hstore, - geometry Geometry(Geometry,4326) NOT NULL""") + address HSTORE, + extratags HSTORE, + geometry GEOMETRY(Geometry,4326) NOT NULL""") @pytest.fixture @@ -223,9 +223,9 @@ def place_postcode_table(temp_db_with_extensions, table_factory): """osm_type char(1) NOT NULL, osm_id bigint NOT NULL, postcode text NOT NULL, - country_code text, - centroid Geometry(Point, 4326) NOT NULL, - geometry Geometry(Geometry, 4326)""") + country_code TEXT, + centroid GEOMETRY(Point, 4326) NOT NULL, + geometry GEOMETRY(Geometry, 4326)""") @pytest.fixture @@ -246,6 +246,36 @@ def place_postcode_row(place_postcode_table, temp_db_cursor): return _insert +@pytest.fixture +def place_interpolation_table(temp_db_with_extensions, table_factory): + """ Create an empty version of the place_interpolation table. + """ + table_factory('place_interpolation', + """osm_type char(1) NOT NULL, + osm_id bigint NOT NULL, + type TEXT, + address HSTORE, + nodes BIGINT[], + geometry GEOMETRY(Geometry, 4326)""") + + +@pytest.fixture +def place_interpolation_row(place_interpolation_table, temp_db_cursor): + """ A factory for rows in the place_interpolation table. The table is created as a + prerequisite to the fixture. + """ + idseq = itertools.count(30001) + + def _insert(osm_type='N', osm_id=None, typ='odd', address=None, + nodes=None, geom='LINESTRING(0.1 0.21, 0.1 0.2)'): + params = {'osm_type': osm_type, 'osm_id': osm_id or next(idseq), + 'type': typ, 'address': address, 'nodes': nodes, + 'geometry': _with_srid(geom)} + temp_db_cursor.insert_row('place_interpolation', **params) + + return _insert + + @pytest.fixture def placex_table(temp_db_with_extensions, temp_db_conn, load_sql, place_table): """ Create an empty version of the placex table. @@ -289,10 +319,11 @@ def osmline_table(temp_db_with_extensions, load_sql): def osmline_row(osmline_table, temp_db_cursor): idseq = itertools.count(20001) - def _add(osm_id=None, geom='LINESTRING(12.0 11.0, 12.003 11.0)'): + def _add(osm_type='W', osm_id=None, geom='LINESTRING(12.0 11.0, 12.003 11.0)'): return temp_db_cursor.insert_row( 'location_property_osmline', place_id=pysql.SQL("nextval('seq_place')"), + osm_type=osm_type, osm_id=osm_id or next(idseq), geometry_sector=pysql.Literal(20), partition=pysql.Literal(0), diff --git a/test/python/tools/test_admin.py b/test/python/tools/test_admin.py index 1097707b..47122663 100644 --- a/test/python/tools/test_admin.py +++ b/test/python/tools/test_admin.py @@ -76,8 +76,8 @@ def test_analyse_indexing_with_osm_id(project_env, placex_row): class TestAdminCleanDeleted: @pytest.fixture(autouse=True) - def setup_polygon_delete(self, project_env, table_factory, place_table, placex_row, - osmline_table, temp_db_cursor, load_sql): + def setup_polygon_delete(self, project_env, table_factory, place_interpolation_table, + placex_row, osmline_table, temp_db_cursor, load_sql): """ Set up place_force_delete function and related tables """ self.project_env = project_env @@ -106,16 +106,24 @@ class TestAdminCleanDeleted: class TEXT NOT NULL, type TEXT NOT NULL, deferred BOOLEAN""") + table_factory('place_interpolation_to_be_deleted', + """osm_id BIGINT, + osm_type CHAR(1)""") table_factory('import_polygon_error', """osm_id BIGINT, osm_type CHAR(1), class TEXT NOT NULL, type TEXT NOT NULL""") - temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION place_delete() - RETURNS TRIGGER AS $$ - BEGIN RETURN NULL; END; - $$ LANGUAGE plpgsql;""") - temp_db_cursor.execute("""CREATE TRIGGER place_before_delete BEFORE DELETE ON place - FOR EACH ROW EXECUTE PROCEDURE place_delete();""") + temp_db_cursor.execute(""" + CREATE OR REPLACE FUNCTION place_delete() RETURNS TRIGGER AS $$ + BEGIN RETURN NULL; END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER place_before_delete BEFORE DELETE ON place + FOR EACH ROW EXECUTE PROCEDURE place_delete(); + + CREATE TRIGGER place_interpolation_before_delete BEFORE DELETE ON place_interpolation + FOR EACH ROW EXECUTE PROCEDURE place_delete(); + """) load_sql('functions/utils.sql') def test_admin_clean_deleted_no_records(self): diff --git a/test/python/tools/test_database_import.py b/test/python/tools/test_database_import.py index 8c0aff8b..274fd529 100644 --- a/test/python/tools/test_database_import.py +++ b/test/python/tools/test_database_import.py @@ -165,12 +165,12 @@ def test_truncate_database_tables(temp_db_conn, temp_db_cursor, table_factory, w @pytest.mark.parametrize("threads", (1, 5)) @pytest.mark.asyncio -async def test_load_data(dsn, place_row, placex_table, osmline_table, +async def test_load_data(dsn, place_row, place_interpolation_row, placex_table, osmline_table, temp_db_cursor, threads): for oid in range(100, 130): place_row(osm_id=oid) - place_row(osm_type='W', osm_id=342, cls='place', typ='houses', - geom='LINESTRING(0 0, 10 10)') + place_interpolation_row(osm_type='W', osm_id=342, typ='odd', + geom='LINESTRING(0 0, 10 10)') temp_db_cursor.execute(""" CREATE OR REPLACE FUNCTION placex_insert() RETURNS TRIGGER AS $$ @@ -238,7 +238,7 @@ class TestSetupSQL: assert not reverse == temp_db_cursor.table_exists('search_name') def test_create_table_triggers(self, temp_db_conn, placex_table, osmline_table, - postcode_table, load_sql): + place_interpolation_table, postcode_table, load_sql): load_sql('functions.sql') database_import.create_table_triggers(temp_db_conn, self.config) From b43116ff522ba76de516c9138ec7823dffcf7d56 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 16 Feb 2026 18:04:47 +0100 Subject: [PATCH 05/10] BDD tests: factor out insert sql code --- test/bdd/test_db.py | 104 ++++++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 38 deletions(-) diff --git a/test/bdd/test_db.py b/test/bdd/test_db.py index 1a7eef7c..68a2a8f2 100644 --- a/test/bdd/test_db.py +++ b/test/bdd/test_db.py @@ -14,6 +14,7 @@ import re from collections import defaultdict import psycopg +import psycopg.sql as pysql import pytest from pytest_bdd import when, then, given @@ -50,6 +51,34 @@ def _collect_place_ids(conn): return pids +@pytest.fixture +def row_factory(db_conn): + def _insert_row(table, **data): + columns = [] + placeholders = [] + values = [] + for k, v in data.items(): + columns.append(pysql.Identifier(k)) + if isinstance(v, tuple): + placeholders.append(pysql.SQL(v[0])) + values.append(v[1]) + elif isinstance(v, (pysql.Literal, pysql.SQL)): + placeholders.append(v) + else: + placeholders.append(pysql.Placeholder()) + values.append(v) + + sql = pysql.SQL("INSERT INTO {table} ({columns}) VALUES({values})")\ + .format(table=pysql.Identifier(table), + columns=pysql.SQL(',').join(columns), + values=pysql.SQL(',').join(placeholders)) + + db_conn.execute(sql, values) + db_conn.commit() + + return _insert_row + + @pytest.fixture def test_config_env(pytestconfig): dbname = pytestconfig.getini('nominatim_test_db') @@ -85,18 +114,19 @@ def import_places(db_conn, named, datatable, node_grid): @given(step_parse('the entrances'), target_fixture=None) -def import_place_entrances(db_conn, datatable, node_grid): +def import_place_entrances(row_factory, datatable, node_grid): """ Insert todo rows into the place_entrance table. """ - with db_conn.cursor() as cur: - for row in datatable[1:]: - data = PlaceColumn(node_grid).add_row(datatable[0], row, False) - assert data.columns['osm_type'] == 'N' + for row in datatable[1:]: + data = PlaceColumn(node_grid).add_row(datatable[0], row, False) + assert data.columns['osm_type'] == 'N' - cur.execute("""INSERT INTO place_entrance (osm_id, type, extratags, geometry) - VALUES (%s, %s, %s, {})""".format(data.get_wkt()), - (data.columns['osm_id'], data.columns['type'], - data.columns.get('extratags'))) + params = {'osm_id': data.columns['osm_id'], + 'type': data.columns['type'], + 'extratags': data.columns.get('extratags'), + 'geometry': pysql.SQL(data.get_wkt())} + + row_factory('place_entrance', **params) @given(step_parse('the postcodes'), target_fixture=None) @@ -135,43 +165,41 @@ def import_place_postcode(db_conn, datatable, node_grid): @given('the ways', target_fixture=None) -def import_ways(db_conn, datatable): +def import_ways(row_factory, datatable): """ Import raw ways into the osm2pgsql way middle table. """ - with db_conn.cursor() as cur: - id_idx = datatable[0].index('id') - node_idx = datatable[0].index('nodes') - for line in datatable[1:]: - tags = psycopg.types.json.Json( - {k[5:]: v for k, v in zip(datatable[0], line) - if k.startswith("tags+")}) - nodes = [int(x) for x in line[node_idx].split(',')] - - cur.execute("INSERT INTO planet_osm_ways (id, nodes, tags) VALUES (%s, %s, %s)", - (line[id_idx], nodes, tags)) + id_idx = datatable[0].index('id') + node_idx = datatable[0].index('nodes') + for line in datatable[1:]: + row_factory('planet_osm_ways', + id=line[id_idx], + nodes=[int(x) for x in line[node_idx].split(',')], + tags=psycopg.types.json.Json( + {k[5:]: v for k, v in zip(datatable[0], line) + if k.startswith("tags+")})) @given('the relations', target_fixture=None) -def import_rels(db_conn, datatable): +def import_rels(row_factory, datatable): """ Import raw relations into the osm2pgsql relation middle table. """ - with db_conn.cursor() as cur: - id_idx = datatable[0].index('id') - memb_idx = datatable[0].index('members') - for line in datatable[1:]: - tags = psycopg.types.json.Json( - {k[5:]: v for k, v in zip(datatable[0], line) - if k.startswith("tags+")}) - members = [] - if line[memb_idx]: - for member in line[memb_idx].split(','): - m = re.fullmatch(r'\s*([RWN])(\d+)(?::(\S+))?\s*', member) - if not m: - raise ValueError(f'Illegal member {member}.') - members.append({'ref': int(m[2]), 'role': m[3] or '', 'type': m[1]}) + id_idx = datatable[0].index('id') + memb_idx = datatable[0].index('members') + for line in datatable[1:]: + tags = psycopg.types.json.Json( + {k[5:]: v for k, v in zip(datatable[0], line) + if k.startswith("tags+")}) + members = [] + if line[memb_idx]: + for member in line[memb_idx].split(','): + m = re.fullmatch(r'\s*([RWN])(\d+)(?::(\S+))?\s*', member) + if not m: + raise ValueError(f'Illegal member {member}.') + members.append({'ref': int(m[2]), 'role': m[3] or '', 'type': m[1]}) - cur.execute('INSERT INTO planet_osm_rels (id, tags, members) VALUES (%s, %s, %s)', - (int(line[id_idx]), tags, psycopg.types.json.Json(members))) + row_factory('planet_osm_rels', + id=int(line[id_idx]), tags=tags, + members=psycopg.types.json.Json(members)) @when('importing', target_fixture='place_ids') From c25204ce31580e3349cdf61b4009f9d2479d536d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 17 Feb 2026 16:29:47 +0100 Subject: [PATCH 06/10] adapt BDD tests to separate interpolation table --- lib-sql/functions/interpolation.sql | 45 ++-- .../bdd/features/db/import/addressing.feature | 10 +- .../features/db/import/interpolation.feature | 225 ++++++------------ test/bdd/features/db/import/linking.feature | 22 +- .../features/db/query/housenumbers.feature | 1 - .../features/db/query/interpolation.feature | 47 +++- .../features/db/update/interpolation.feature | 197 ++++----------- test/bdd/features/db/update/simple.feature | 13 - .../features/osm2pgsql/import/tags.feature | 16 +- .../osm2pgsql/update/interpolations.feature | 38 ++- .../osm2pgsql/update/postcodes.feature | 12 +- test/bdd/test_db.py | 49 +++- test/bdd/utils/place_inserter.py | 1 - 13 files changed, 290 insertions(+), 386 deletions(-) diff --git a/lib-sql/functions/interpolation.sql b/lib-sql/functions/interpolation.sql index a31c1793..d558378d 100644 --- a/lib-sql/functions/interpolation.sql +++ b/lib-sql/functions/interpolation.sql @@ -69,16 +69,13 @@ BEGIN -- finally update/insert place_interpolation itself IF existing.osm_type is not NULL THEN - IF existing.type != NEW.type - OR coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore) - OR existing.geometry::text != NEW.geometry::text - THEN - UPDATE place_interpolation p - SET type = NEW.type, - address = NEW.address, - geometry = NEW.geometry - WHERE p.osm_type = NEW.osm_type AND p.osm_id = NEW.osm_id; - END IF; + -- Always updates as the nodes with the housenumber might be the reason + -- for the change. + UPDATE place_interpolation p + SET type = NEW.type, + address = NEW.address, + geometry = NEW.geometry + WHERE p.osm_type = NEW.osm_type AND p.osm_id = NEW.osm_id; RETURN NULL; END IF; @@ -114,17 +111,21 @@ BEGIN RETURN in_address; END IF; - SELECT nodes INTO waynodes FROM planet_osm_ways WHERE id = wayid; - FOR location IN - SELECT placex.address, placex.osm_id FROM placex - WHERE osm_type = 'N' and osm_id = ANY(waynodes) - and placex.address is not null - and (placex.address ? 'street' or placex.address ? 'place') - and indexed_status < 100 - LOOP - -- mark it as a derived address - RETURN location.address || coalesce(in_address, '{}'::hstore) || hstore('_inherited', ''); - END LOOP; + SELECT nodes INTO waynodes FROM place_interpolation + WHERE osm_type = 'W' AND osm_id = wayid; + + IF array_upper(waynodes, 1) IS NOT NULL THEN + FOR location IN + SELECT placex.address, placex.osm_id FROM placex + WHERE osm_type = 'N' and osm_id = ANY(waynodes) + and placex.address is not null + and (placex.address ? 'street' or placex.address ? 'place') + and indexed_status < 100 + LOOP + -- mark it as a derived address + RETURN location.address || coalesce(in_address, ''::hstore) || hstore('_inherited', ''); + END LOOP; + END IF; RETURN in_address; END; @@ -285,7 +286,7 @@ BEGIN ELSE -- classic interpolation way SELECT nodes INTO waynodes - FROM planet_osm_ways WHERE id = NEW.osm_id; + FROM place_interpolation WHERE osm_type = NEW.osm_type AND osm_id = NEW.osm_id; IF array_upper(waynodes, 1) IS NULL THEN RETURN NEW; diff --git a/test/bdd/features/db/import/addressing.feature b/test/bdd/features/db/import/addressing.feature index 82ca1e6f..e4f10b6b 100644 --- a/test/bdd/features/db/import/addressing.feature +++ b/test/bdd/features/db/import/addressing.feature @@ -241,8 +241,8 @@ Feature: Address computation Scenario: buildings with only addr:postcodes do not appear in the address of a way Given the grid with origin DE | 1 | | | | | 8 | | 6 | | 2 | - | |10 |11 | | | | | | | | - | |13 |12 | | | | | | | | + | | | | | | | | | | | + | |13 | | | | | | | | | | 20| | | 21| | | | | | | | | | | | | | | | | | | | | | | | 9 | | | | | @@ -255,9 +255,9 @@ Feature: Address computation And the named places | osm | class | type | geometry | | W93 | highway | residential | 20,21 | - And the places - | osm | class | type | addr+postcode | geometry | - | W22 | place | postcode | 11234 | (10,11,12,13,10) | + And the postcodes + | osm | postcode | centroid | + | W22 | 11234 | 13 | When importing Then place_addressline contains exactly | object | address | diff --git a/test/bdd/features/db/import/interpolation.feature b/test/bdd/features/db/import/interpolation.feature index b1f31f8f..cc332852 100644 --- a/test/bdd/features/db/import/interpolation.feature +++ b/test/bdd/features/db/import/interpolation.feature @@ -8,12 +8,9 @@ Feature: Import of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,2 | - And the ways - | id | nodes | - | 1 | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,2 | 1,2 | When importing Then W1 expands to no interpolation @@ -25,15 +22,12 @@ Feature: Import of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,2 | 1,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 1,2 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -47,15 +41,12 @@ Feature: Import of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 8 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 2,1 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 2,1 | 2,1 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 2,1 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -69,15 +60,12 @@ Feature: Import of address interpolations | osm | class | type | housenr | | N1 | place | house | 1 | | N2 | place | house | 11 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | odd | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | odd | 1,2 | 1,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 1,2 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -91,15 +79,12 @@ Feature: Import of address interpolations | osm | class | type | housenr | | N1 | place | house | 1 | | N2 | place | house | 4 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | all | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | all | 1,2 | 1,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 1,2 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -113,15 +98,12 @@ Feature: Import of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 12 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,3,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,3,2 | 1,3,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 1,3,2 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -135,15 +117,12 @@ Feature: Import of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 10 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,3,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,3,2 | 1,3,3,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 1,3,3,2 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -158,15 +137,12 @@ Feature: Import of address interpolations | N1 | place | house | 2 | | N2 | place | house | 14 | | N3 | place | house | 10 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,3,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,3,2 | 1,3,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 1,3,2 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -184,15 +160,12 @@ Feature: Import of address interpolations | N2 | place | house | 14 | | N3 | place | house | 10 | | N4 | place | house | 18 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,3,2,4 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,3,2,4 | 1,3,2,4 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 1,3,2,4 | - And the ways - | id | nodes | - | 1 | 1,3,2,4 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -209,15 +182,12 @@ Feature: Import of address interpolations | N1 | place | house | 2 | | N2 | place | house | 14 | | N3 | place | house | 10 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 2,3,1 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 2,3,1 | 2,3,1 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 2,3,1 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -233,15 +203,12 @@ Feature: Import of address interpolations | N1 | place | house | 2 | | N2 | place | house | 8 | | N3 | place | house | 7 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,3,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,3,2 | 1,3,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 1,3,2 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -257,15 +224,12 @@ Feature: Import of address interpolations | N1 | place | house | 2 | | N2 | place | house | 6 | | N3 | place | house | 10 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,2,3,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,2,3,2 | 1,2,3,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 1,2,3 | - And the ways - | id | nodes | - | 1 | 1,2,3,2 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -281,15 +245,12 @@ Feature: Import of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,2,3,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,2,3,2 | 1,2,3,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 1,2,3 | - And the ways - | id | nodes | - | 1 | 1,2,3,2 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -306,18 +267,14 @@ Feature: Import of address interpolations | N2 | place | house | 6 | 2 | | N3 | place | house | 12 | 1 | | N4 | place | house | 16 | 2 | - And the places - | osm | class | type | addr+interpolation | street | geometry | - | W10 | place | houses | even | | 1,2 | - | W11 | place | houses | even | Cloud Street | 1,2 | + And the interpolations + | osm | type | street | nodes | geometry | nodes | + | W10 | even | | 1,2 | 1,2 | 1,2 | + | W11 | even | Cloud Street | 3,4 | 1,2 | 3,4 | And the places | osm | class | type | name | geometry | | W2 | highway | tertiary | Sun Way | 10,11 | | W3 | highway | tertiary | Cloud Street | 20,21 | - And the ways - | id | nodes | - | 10 | 1,2 | - | 11 | 3,4 | When importing Then placex contains | object | parent_place_id | @@ -351,18 +308,14 @@ Feature: Import of address interpolations | N2 | place | house | 6 | | 2 | | N3 | place | house | 12 | Cloud Street | 1 | | N4 | place | house | 16 | Cloud Street | 2 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W10 | place | houses | even | 1,2 | - | W11 | place | houses | even | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W10 | even | 1,2 | 1,2 | + | W11 | even | 1,2 | 3,4 | And the places | osm | class | type | name | geometry | | W2 | highway | tertiary | Sun Way | 10,11 | | W3 | highway | tertiary | Cloud Street | 20,21 | - And the ways - | id | nodes | - | 10 | 1,2 | - | 11 | 3,4 | When importing Then placex contains | object | parent_place_id | @@ -391,15 +344,12 @@ Feature: Import of address interpolations | N1 | place | house | 10 | 144.9632341 -37.76163 | | N2 | place | house | 6 | 144.9630541 -37.7628174 | | N3 | shop | supermarket | 2 | 144.9629794 -37.7630755 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 144.9632341 -37.76163,144.9630541 -37.7628172,144.9629794 -37.7630755 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 144.9632341 -37.76163,144.9630541 -37.7628172,144.9629794 -37.7630755 | 1,2,3 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 144.9632341 -37.76163,144.9629794 -37.7630755 | - And the ways - | id | nodes | - | 1 | 1,2,3 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -415,24 +365,21 @@ Feature: Import of address interpolations | N1 | place | house | 23 | | N2 | amenity | school | | | N3 | place | house | 29 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | odd | 1,2,3 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | odd | 1,2,3 | 1,2,3 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 1,2,3 | When importing Then W1 expands to interpolation | start | end | geometry | | 25 | 27 | 0.0000166 0,0.00002 0,0.0000333 0 | Scenario: Ways without node entries are ignored - Given the places - | osm | class | type | housenr | geometry | - | W1 | place | houses | even | 1 1, 1 1.001 | + Given the interpolations + | osm | type | geometry | + | W1 | even | 1 1, 1 1.001 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 1 1, 1 1.001 | @@ -447,9 +394,9 @@ Feature: Import of address interpolations | osm | class | type | | N1 | place | house | | N2 | place | house | - Given the places - | osm | class | type | housenr | geometry | - | W1 | place | houses | even | 1,2 | + Given the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,2 | 1,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | @@ -464,15 +411,12 @@ Feature: Import of address interpolations | osm | class | type | housenr | | N1 | place | house | 0 | | N2 | place | house | 10 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,2 | 1,2 | And the places | osm | class | type | name | geometry | | W10 | highway | residential | London Road |4,5 | - And the ways - | id | nodes | - | 1 | 1,2 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -497,12 +441,9 @@ Feature: Import of address interpolations | osm | class | type | name | geometry | | W1 | highway | residential | Vert St | 1,2 | | W2 | highway | residential | Horiz St | 2,3 | - And the places - | osm | class | type | addr+interpolation | addr+inclusion | geometry | - | W10 | place | houses | even | actual | 8,9 | - And the ways - | id | nodes | - | 10 | 8,9 | + And the interpolations + | osm | type | addr+inclusion | geometry | nodes | + | W10 | even | actual | 8,9 | 8,9 | When importing Then placex contains | object | parent_place_id | @@ -521,15 +462,12 @@ Feature: Import of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | | 1,2 | 1,2 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 1,2 | When importing Then W1 expands to no interpolation @@ -549,15 +487,12 @@ Feature: Import of address interpolations | N2 | place | house | 18 | 3 | | N3 | place | house | 24 | 9 | | N4 | place | house | 42 | 4 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,2,3,4 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,2,3,4 | 1,2,3,4 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 1,4 | - And the ways - | id | nodes | - | 1 | 1,2,3,4 | When importing Then W1 expands to interpolation | start | end | @@ -576,15 +511,12 @@ Feature: Import of address interpolations | N2 | place | house | 6 | 8 | | N3 | place | house | 10 | 8 | | N4 | place | house | 14 | 9 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 7,8,8,9 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 7,8,8,9 | 1,2,3,4 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 4,5 | - And the ways - | id | nodes | - | 1 | 1,2,3,4 | When importing Then W1 expands to interpolation | start | end | geometry | @@ -601,15 +533,12 @@ Feature: Import of address interpolations | N2 | place | house | 8 | | N3 | place | house | 12 | | N4 | place | house | 14 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 8,9 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 8,9 | 1,8,9,2,3,4 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 1,4 | - And the ways - | id | nodes | - | 1 | 1,8,9,2,3,4 | When importing Then W1 expands to interpolation | start | end | geometry | diff --git a/test/bdd/features/db/import/linking.feature b/test/bdd/features/db/import/linking.feature index 29c11447..dbaac5fc 100644 --- a/test/bdd/features/db/import/linking.feature +++ b/test/bdd/features/db/import/linking.feature @@ -297,25 +297,21 @@ Feature: Linking of places | R1 | LabelPlace | + @skip Scenario: Linked places expand default language names - Given the grid with origin CO - | 1 | | 2 | | 5 | | 6 | - | | 9 | | | | 10 | | - | 4 | | 3 | | 8 | | 7 | - And the places + Given the grid + | 1 | | 2 | + | | 9 | | + | 4 | | 3 | + Given the places | osm | class | type | name+name | geometry | | N9 | place | city | Popayán | 9 | - And the places - | osm | class | type | name+name:en | geometry | - | N10 | place | city | Open | 10 | - And the places + Given the places | osm | class | type | name+name | geometry | admin | | R1 | boundary | administrative | Perímetro Urbano Popayán | (1,2,3,4,1) | 8 | - | R2 | boundary | administrative | Abre | (5,6,7,8,5) | 8 | And the relations - | id | members | - | 1 | N9:label | - | 2 | N10:label | + | id | members | + | 1 | N9:label | When importing Then placex contains | object | linked_place_id | diff --git a/test/bdd/features/db/query/housenumbers.feature b/test/bdd/features/db/query/housenumbers.feature index 75d77b18..df466ecb 100644 --- a/test/bdd/features/db/query/housenumbers.feature +++ b/test/bdd/features/db/query/housenumbers.feature @@ -378,4 +378,3 @@ Feature: Searching of house numbers Then the result set contains | object | | W10 | - diff --git a/test/bdd/features/db/query/interpolation.feature b/test/bdd/features/db/query/interpolation.feature index 1746d37d..a893721f 100644 --- a/test/bdd/features/db/query/interpolation.feature +++ b/test/bdd/features/db/query/interpolation.feature @@ -11,16 +11,13 @@ Feature: Query of address interpolations Given the places | osm | class | type | name | geometry | | W10 | highway | primary | Nickway | 10,12,13 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | odd | 1,3 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | odd | 1,3 | 1,3 | And the places | osm | class | type | housenr | geometry | | N1 | place | house | 1 | 1 | | N3 | place | house | 5 | 3 | - And the ways - | id | nodes | - | 1 | 1,3 | When importing When reverse geocoding at node 2 Then the result contains @@ -36,16 +33,13 @@ Feature: Query of address interpolations Given the places | osm | class | type | name | geometry | | W10 | highway | primary | Nickway | 10,12,13 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,3 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,3 | 1,3 | And the places | osm | class | type | housenr | geometry | | N1 | place | house | 2 | 1 | | N3 | place | house | 18 | 3 | - And the ways - | id | nodes | - | 1 | 1,3 | When importing When reverse geocoding at node 2 Then the result contains @@ -55,3 +49,32 @@ Feature: Query of address interpolations Then all results contain | object | display_name | centroid!wkt | | W1 | 10, Nickway | 2 | + + + Scenario: Interpolations are found according to their type + Given the grid + | 10 | | 11 | + | 100 | | 101 | + | 20 | | 21 | + And the places + | osm | class | type | name | geometry | + | W100 | highway | residential | Ringstr | 100, 101 | + And the interpolations + | osm | type | geometry | nodes | + | W10 | even | 10, 11 | 10, 11 | + | W20 | odd | 20, 21 | 20, 21 | + And the places + | osm | class | type | housenr | geometry | + | N10 | place | house | 10 | 10 | + | N11 | place | house | 20 | 11 | + | N20 | place | house | 11 | 20 | + | N21 | place | house | 21 | 21 | + When importing + When geocoding "Ringstr 12" + Then the result set contains + | object | + | W10 | + When geocoding "Ringstr 13" + Then the result set contains + | object | + | W20 | diff --git a/test/bdd/features/db/update/interpolation.feature b/test/bdd/features/db/update/interpolation.feature index e548862b..e68ef159 100644 --- a/test/bdd/features/db/update/interpolation.feature +++ b/test/bdd/features/db/update/interpolation.feature @@ -11,18 +11,15 @@ Feature: Update of address interpolations | osm | class | type | name | geometry | | W2 | highway | unclassified | Sun Way | 10,11 | | W3 | highway | unclassified | Cloud Street | 20,21 | - And the ways - | id | nodes | - | 10 | 1,2 | When importing Then W10 expands to no interpolation When updating places | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | - And updating places - | osm | class | type | addr+interpolation | geometry | - | W10 | place | houses | even | 1,2 | + And updating interpolations + | osm | type | geometry | nodes | + | W10 | even | 1,2 | 1,2 | Then placex contains | object | parent_place_id | | N1 | W2 | @@ -41,16 +38,13 @@ Feature: Update of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W10 | place | houses | even | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W10 | even | 1,2 | 1,2 | And the places | osm | class | type | name | geometry | | W2 | highway | unclassified | Sun Way | 10,11 | | W3 | highway | unclassified | Cloud Street | 20,21 | - And the ways - | id | nodes | - | 10 | 1,2 | When importing Then placex contains | object | parent_place_id | @@ -59,9 +53,9 @@ Feature: Update of address interpolations And W10 expands to interpolation | parent_place_id | start | end | | W2 | 4 | 4 | - When updating places - | osm | class | type | addr+interpolation | street | geometry | - | W10 | place | houses | even | Cloud Street | 1,2 | + When updating interpolations + | osm | type | street | nodes | geometry | + | W10 | even | Cloud Street | 1,2 | 1,2 | Then placex contains | object | parent_place_id | | N1 | W3 | @@ -80,16 +74,13 @@ Feature: Update of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W10 | place | houses | even | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W10 | even | 1,2 | 1,2 | And the places | osm | class | type | name | geometry | | W2 | highway | unclassified | Sun Way | 10,11 | | W3 | highway | unclassified | Cloud Street | 20,21 | - And the ways - | id | nodes | - | 10 | 1,2 | When importing Then placex contains | object | parent_place_id | @@ -120,16 +111,13 @@ Feature: Update of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W10 | place | houses | even | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W10 | even | 1,2 | 1,2 | And the places | osm | class | type | name | geometry | | W2 | highway | unclassified | Sun Way | 10,11 | | W3 | highway | unclassified | Cloud Street | 20,21 | - And the ways - | id | nodes | - | 10 | 1,2 | When importing Then placex contains | object | parent_place_id | @@ -155,15 +143,12 @@ Feature: Update of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | - And the places - | osm | class | type | addr+interpolation | street | geometry | - | W10 | place | houses | even | Cloud Street| 1,2 | + And the interpolations + | osm | type | street | geometry | nodes | + | W10 | even | Cloud Street| 1,2 | 1,2 | And the places | osm | class | type | name | geometry | | W2 | highway | unclassified | Sun Way | 10,11 | - And the ways - | id | nodes | - | 10 | 1,2 | When importing Then placex contains | object | parent_place_id | @@ -193,16 +178,13 @@ Feature: Update of address interpolations | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | - And the places - | osm | class | type | addr+interpolation | street | geometry | - | W10 | place | houses | even | Cloud Street| 1,2 | + And the interpolations + | osm | type | street | geometry | nodes | + | W10 | even | Cloud Street| 1,2 | 1,2 | And the places | osm | class | type | name | geometry | | W2 | highway | unclassified | Sun Way | 10,11 | | W3 | highway | unclassified | Cloud Street | 20,21 | - And the ways - | id | nodes | - | 10 | 1,2 | When importing Then placex contains | object | parent_place_id | @@ -220,67 +202,6 @@ Feature: Update of address interpolations | parent_place_id | start | end | | W2 | 4 | 4 | - Scenario: building becomes interpolation - Given the grid - | 10 | | | | 11 | - | | 1 | | 2 | | - | | 4 | | 3 | | - And the places - | osm | class | type | housenr | geometry | - | W1 | place | house | 3 | (1,2,3,4,1) | - And the places - | osm | class | type | name | geometry | - | W2 | highway | unclassified | Cloud Street | 10,11 | - When importing - Then placex contains - | object | parent_place_id | - | W1 | W2 | - Given the ways - | id | nodes | - | 1 | 1,2 | - When updating places - | osm | class | type | housenr | - | N1 | place | house | 2 | - | N2 | place | house | 6 | - And updating places - | osm | class | type | addr+interpolation | street | geometry | - | W1 | place | houses | even | Cloud Street| 1,2 | - Then placex has no entry for W1 - And W1 expands to interpolation - | parent_place_id | start | end | - | W2 | 4 | 4 | - - Scenario: interpolation becomes building - Given the grid - | 10 | | | | 11 | - | | 1 | | 2 | | - | | 4 | | 3 | | - And the places - | osm | class | type | housenr | - | N1 | place | house | 2 | - | N2 | place | house | 6 | - And the places - | osm | class | type | name | geometry | - | W2 | highway | unclassified | Cloud Street | 10,11 | - And the ways - | id | nodes | - | 1 | 1,2 | - And the places - | osm | class | type | addr+interpolation | street | geometry | - | W1 | place | houses | even | Cloud Street| 1,2 | - When importing - Then placex has no entry for W1 - And W1 expands to interpolation - | parent_place_id | start | end | - | W2 | 4 | 4 | - When updating places - | osm | class | type | housenr | geometry | - | W1 | place | house | 3 | (1,2,3,4,1) | - Then placex contains - | object | parent_place_id | - | W1 | W2 | - And W1 expands to no interpolation - Scenario: housenumbers added to interpolation Given the grid | 10 | | | | 11 | @@ -288,18 +209,18 @@ Feature: Update of address interpolations And the places | osm | class | type | name | geometry | | W2 | highway | unclassified | Cloud Street | 10,11 | - And the ways - | id | nodes | - | 1 | 1,2 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W1 | place | houses | even | 1,2 | + And the interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,2 | 1,2 | When importing Then W1 expands to no interpolation When updating places | osm | class | type | housenr | | N1 | place | house | 2 | | N2 | place | house | 6 | + And updating interpolations + | osm | type | geometry | nodes | + | W1 | even | 1,2 | 1,2 | Then W1 expands to interpolation | parent_place_id | start | end | | W2 | 4 | 4 | @@ -311,12 +232,9 @@ Feature: Update of address interpolations And the places | osm | class | type | name | geometry | | W1 | highway | unclassified | Cloud Street | 1, 2 | - And the ways - | id | nodes | - | 2 | 3,4,5 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W2 | place | houses | even | 3,4,5 | + And the interpolations + | osm | type | geometry | nodes | + | W2 | even | 3,4,5 | 3,4,5 | And the places | osm | class | type | housenr | | N3 | place | house | 2 | @@ -328,12 +246,14 @@ Feature: Update of address interpolations When updating places | osm | class | type | housenr | | N4 | place | house | 6 | + And updating interpolations + | osm | type | geometry | nodes | + | W2 | even | 3,4,5 | 3,4,5 | Then W2 expands to interpolation | parent_place_id | start | end | | W1 | 4 | 4 | | W1 | 8 | 8 | - @skip Scenario: housenumber removed in middle of interpolation Given the grid | 1 | | | | | 2 | @@ -341,12 +261,9 @@ Feature: Update of address interpolations And the places | osm | class | type | name | geometry | | W1 | highway | unclassified | Cloud Street | 1, 2 | - And the ways - | id | nodes | - | 2 | 3,4,5 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W2 | place | houses | even | 3,4,5 | + And the interpolations + | osm | type | geometry | nodes | + | W2 | even | 3,4,5 | 3,4,5 | And the places | osm | class | type | housenr | | N3 | place | house | 2 | @@ -358,6 +275,9 @@ Feature: Update of address interpolations | W1 | 4 | 4 | | W1 | 8 | 8 | When marking for delete N4 + And updating interpolations + | osm | type | geometry | nodes | + | W2 | even | 3,4,5 | 3,4,5 | Then W2 expands to interpolation | parent_place_id | start | end | | W1 | 4 | 8 | @@ -369,12 +289,9 @@ Feature: Update of address interpolations And the places | osm | class | type | name | geometry | | W1 | highway | unclassified | Cloud Street | 1, 2 | - And the ways - | id | nodes | - | 2 | 3,4 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W2 | place | houses | even | 3,4 | + And the interpolations + | osm | type | geometry | nodes | + | W2 | even | 3,4 | 3,4 | And the places | osm | class | type | housenr | | N3 | place | house | 2 | @@ -386,33 +303,9 @@ Feature: Update of address interpolations When updating places | osm | class | type | housenr | | N4 | place | house | 8 | + And updating interpolations + | osm | type | geometry | nodes | + | W2 | even | 3,4 | 3,4 | Then W2 expands to interpolation | parent_place_id | start | end | | W1 | 4 | 6 | - - Scenario: Legal interpolation type changed to illegal one - Given the grid - | 1 | | 2 | - | 3 | | 4 | - And the places - | osm | class | type | name | geometry | - | W1 | highway | unclassified | Cloud Street | 1, 2 | - And the ways - | id | nodes | - | 2 | 3,4 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W2 | place | houses | even | 3,4 | - And the places - | osm | class | type | housenr | - | N3 | place | house | 2 | - | N4 | place | house | 6 | - When importing - Then W2 expands to interpolation - | parent_place_id | start | end | - | W1 | 4 | 4 | - When updating places - | osm | class | type | addr+interpolation | geometry | - | W2 | place | houses | 12-2 | 3,4 | - Then W2 expands to no interpolation - diff --git a/test/bdd/features/db/update/simple.feature b/test/bdd/features/db/update/simple.feature index 22165c2f..2a9a86fa 100644 --- a/test/bdd/features/db/update/simple.feature +++ b/test/bdd/features/db/update/simple.feature @@ -68,19 +68,6 @@ Feature: Update of simple objects | object | class | type | centroid!wkt | | N3 | shop | grocery | 1 -1 | - Scenario: remove postcode place when house number is added - Given the places - | osm | class | type | postcode | geometry | - | N3 | place | postcode | 12345 | country:de | - When importing - Then placex has no entry for N3 - When updating places - | osm | class | type | postcode | housenr | geometry | - | N3 | place | house | 12345 | 13 | country:de | - Then placex contains - | object | class | type | - | N3 | place | house | - Scenario: remove boundary when changing from polygon to way Given the grid | 1 | 2 | diff --git a/test/bdd/features/osm2pgsql/import/tags.feature b/test/bdd/features/osm2pgsql/import/tags.feature index cb1fab6a..b59da688 100644 --- a/test/bdd/features/osm2pgsql/import/tags.feature +++ b/test/bdd/features/osm2pgsql/import/tags.feature @@ -206,15 +206,21 @@ Feature: Tag evaluation Scenario: Address interpolations + Given the grid + | 1 | 2 | When loading osm data """ - n13001 Taddr:interpolation=odd - n13002 Taddr:interpolation=even,place=city + n1 + n2 + w13001 Taddr:interpolation=odd Nn1,n2 + w13002 Taddr:interpolation=even,place=city Nn1,n2 """ Then place contains exactly - | object | class | type | address!dict | - | N13001 | place | houses | 'interpolation': 'odd' | - | N13002 | place | houses | 'interpolation': 'even' | + | object | + And place_interpolation contains exactly + | object | type | address!dict | + | W13001 | odd | - | + | W13002 | even | - | Scenario: Footways diff --git a/test/bdd/features/osm2pgsql/update/interpolations.feature b/test/bdd/features/osm2pgsql/update/interpolations.feature index ca87ed12..6c76c91b 100644 --- a/test/bdd/features/osm2pgsql/update/interpolations.feature +++ b/test/bdd/features/osm2pgsql/update/interpolations.feature @@ -14,20 +14,24 @@ Feature: Updates of address interpolation objects n2 Taddr:housenumber=17 w33 Thighway=residential,name=Tao Nn1,n2 """ - Then place contains + Then place contains exactly | object | class | type | | N1 | place | house | | N2 | place | house | + | W33 | highway | residential | When updating osm data """ w99 Taddr:interpolation=odd Nn1,n2 """ - Then place contains + Then place contains exactly | object | class | type | | N1 | place | house | | N2 | place | house | - | W99 | place | houses | + | W33 | highway | residential | + And place_interpolation contains exactly + | object | type | + | W99 | odd | When indexing Then placex contains exactly | object | class | type | @@ -46,11 +50,13 @@ Feature: Updates of address interpolation objects n2 Taddr:housenumber=7 w99 Taddr:interpolation=odd Nn1,n2 """ - Then place contains + Then place contains exactly | object | class | type | | N1 | place | house | | N2 | place | house | - | W99 | place | houses | + And place_interpolation contains exactly + | object | type | + | W99 | odd | When updating osm data """ @@ -60,6 +66,8 @@ Feature: Updates of address interpolation objects | object | class | type | | N1 | place | house | | N2 | place | house | + And place_interpolation contains exactly + | object | When indexing Then placex contains exactly | object | class | type | @@ -77,21 +85,27 @@ Feature: Updates of address interpolation objects w33 Thighway=residential Nn1,n2 w99 Thighway=residential Nn1,n2 """ - Then place contains + Then place contains exactly | object | class | type | | N1 | place | house | | N2 | place | house | + | W33 | highway | residential | | W99 | highway | residential | + And place_interpolation contains exactly + | object | When updating osm data """ w99 Taddr:interpolation=odd Nn1,n2 """ - Then place contains + Then place contains exactly | object | class | type | | N1 | place | house | | N2 | place | house | - | W99 | place | houses | + | W33 | highway | residential | + And place_interpolation contains exactly + | object | type | + | W99 | odd | When indexing Then placex contains exactly | object | class | type | @@ -110,11 +124,13 @@ Feature: Updates of address interpolation objects n2 Taddr:housenumber=17 w99 Taddr:interpolation=odd Nn1,n2 """ - Then place contains + Then place contains exactly | object | class | type | | N1 | place | house | | N2 | place | house | - | W99 | place | houses | + And place_interpolation contains exactly + | object | type | + | W99 | odd | When updating osm data """ @@ -125,6 +141,8 @@ Feature: Updates of address interpolation objects | N1 | place | house | | N2 | place | house | | W99 | highway | residential | + And place_interpolation contains exactly + | object | When indexing Then placex contains exactly | object | class | type | diff --git a/test/bdd/features/osm2pgsql/update/postcodes.feature b/test/bdd/features/osm2pgsql/update/postcodes.feature index df0fdb2a..3ae73f27 100644 --- a/test/bdd/features/osm2pgsql/update/postcodes.feature +++ b/test/bdd/features/osm2pgsql/update/postcodes.feature @@ -112,7 +112,9 @@ Feature: Update of postcode only objects | object | class | type | | N1 | place | house | | N2 | place | house | - | W34 | place | houses | + And place_interpolation contains exactly + | object | type | + | W34 | odd | When updating osm data """ @@ -122,9 +124,11 @@ Feature: Update of postcode only objects | object | class | type | | N1 | place | house | | N2 | place | house | - Then place_postcode contains exactly + And place_postcode contains exactly | object | postcode | | W34 | 4456 | + And place_interpolation contains exactly + | object | When indexing Then location_property_osmline contains exactly | osm_id | @@ -158,7 +162,9 @@ Feature: Update of postcode only objects | N1 | place | house | | N2 | place | house | | W33 | highway | residential | - | W34 | place | houses | + And place_interpolation contains exactly + | object | type | + | W34 | odd | And place_postcode contains exactly | object | When indexing diff --git a/test/bdd/test_db.py b/test/bdd/test_db.py index 68a2a8f2..91b5b922 100644 --- a/test/bdd/test_db.py +++ b/test/bdd/test_db.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Collector for BDD import acceptance tests. @@ -129,6 +129,26 @@ def import_place_entrances(row_factory, datatable, node_grid): row_factory('place_entrance', **params) +@given(step_parse('the interpolations'), target_fixture=None) +def import_place_interpolations(row_factory, datatable, node_grid): + """ Insert todo rows into the place_entrance table. + """ + for row in datatable[1:]: + data = PlaceColumn(node_grid).add_row(datatable[0], row, False) + + if 'nodes' in data.columns: + nodes = [int(x) for x in data.columns['nodes'].split(',')] + else: + nodes = None + + params = {'osm_type': data.columns['osm_type'], 'osm_id': data.columns['osm_id'], + 'type': data.columns['type'], + 'address': data.columns.get('address'), 'nodes': nodes, + 'geometry': pysql.SQL(data.get_wkt())} + + row_factory('place_interpolation', **params) + + @given(step_parse('the postcodes'), target_fixture=None) def import_place_postcode(db_conn, datatable, node_grid): """ Insert todo rows into the place_postcode table. If a row for the @@ -249,6 +269,31 @@ def update_place_entrances(db_conn, datatable, node_grid): db_conn.commit() +@when('updating interpolations', target_fixture=None) +def update_place_interpolations(db_conn, row_factory, update_config, datatable, node_grid): + """ Update rows in the place_entrance table. + """ + for row in datatable[1:]: + data = PlaceColumn(node_grid).add_row(datatable[0], row, False) + + if 'nodes' in data.columns: + nodes = [int(x) for x in data.columns['nodes'].split(',')] + else: + nodes = None + + params = {'osm_type': data.columns['osm_type'], 'osm_id': data.columns['osm_id'], + 'type': data.columns['type'], + 'address': data.columns.get('address'), 'nodes': nodes, + 'geometry': pysql.SQL(data.get_wkt())} + + row_factory('place_interpolation', **params) + + db_conn.execute('SELECT flush_deleted_places()') + db_conn.commit() + + cli.nominatim(['index', '-q', '--minrank', '30'], update_config.environ) + + @when('refreshing postcodes') def do_postcode_update(update_config): """ Recompute the postcode centroids. @@ -265,6 +310,8 @@ def do_delete_place(db_conn, update_config, node_grid, otype, oid): cur.execute('TRUNCATE place_to_be_deleted') cur.execute('DELETE FROM place WHERE osm_type = %s and osm_id = %s', (otype, oid)) + cur.execute('DELETE FROM place_interpolation WHERE osm_type = %s and osm_id = %s', + (otype, oid)) cur.execute('SELECT flush_deleted_places()') if otype == 'N': cur.execute('DELETE FROM place_entrance WHERE osm_id = %s', diff --git a/test/bdd/utils/place_inserter.py b/test/bdd/utils/place_inserter.py index 6d37b850..e91e5907 100644 --- a/test/bdd/utils/place_inserter.py +++ b/test/bdd/utils/place_inserter.py @@ -54,7 +54,6 @@ class PlaceColumn: elif key in ('name', 'address', 'extratags'): self.columns[key] = ast.literal_eval('{' + value + '}') else: - assert key in ('class', 'type'), "Unknown column '{}'.".format(key) self.columns[key] = None if value == '' else value def _set_key_name(self, value): From b71543b03bcda38df1ffe53cdb675259b710d353 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 17 Feb 2026 19:52:52 +0100 Subject: [PATCH 07/10] add test for new interpolation variants --- lib-lua/themes/nominatim/init.lua | 23 +++++----- .../osm2pgsql/import/interpolation.feature | 44 +++++++++++++++++++ .../features/osm2pgsql/import/tags.feature | 18 -------- test/bdd/utils/checks.py | 6 ++- 4 files changed, 61 insertions(+), 30 deletions(-) create mode 100644 test/bdd/features/osm2pgsql/import/interpolation.feature diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index 41eaafe1..fd15968b 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -387,7 +387,6 @@ function Place:grab_address_parts(data) if atype ~= nil then if atype == 'main' then - self.has_name = true self.address[strip_address_prefix(k)] = v count = count + 1 elseif atype == 'extra' then @@ -708,24 +707,22 @@ function module.process_tags(o) local fallback = o:grab_name_parts{groups=NAME_FILTER} -- address keys - if o:grab_address_parts{groups=ADDRESS_FILTER} > 0 and fallback == nil then - fallback = {'place', 'house', address_fallback} - end + local fallback_address = (o:grab_address_parts{groups=ADDRESS_FILTER} > 0) if o.address.country ~= nil and #o.address.country ~= 2 then o.address['country'] = nil end if o.address.interpolation ~= nil - and (o.address.housenumber == nil or string.find(o.address.housenumber, '-') ~= nil) then + and (o.address.housenumber == nil + or string.find(o.address.housenumber, '^%d+%-%d+$') ~= nil) then if o:geometry_is_valid() then local extra_addr = nil for k, v in pairs(o.address) do if k ~= 'interpolation' then if extra_addr == nil then - extra_addr = {k = v} - else - extra_addr[k] = v + extra_addr = {} end + extra_addr[k] = v end end local nodes = nil @@ -738,8 +735,10 @@ function module.process_tags(o) nodes = nodes, geometry = o.geometry } + fallback_address = false + else + return end - return end -- collect main keys @@ -761,8 +760,8 @@ function module.process_tags(o) geometry = o.geometry } end - elseif ktype == 'fallback' and o.has_name then - fallback = {k, v, PlaceTransform.named} + elseif ktype == 'fallback' and (fallback_address or o.has_name) then + fallback = {k, v, PlaceTransform.always} end end end @@ -770,6 +769,8 @@ function module.process_tags(o) if o.num_entries == 0 then if fallback ~= nil then o:write_place(fallback[1], fallback[2], fallback[3]) + elseif fallback_address then + o:write_place('place', 'house', address_fallback) elseif POSTCODE_FALLBACK and not postcode_collect and o.address.postcode ~= nil and o:geometry_is_valid() then diff --git a/test/bdd/features/osm2pgsql/import/interpolation.feature b/test/bdd/features/osm2pgsql/import/interpolation.feature new file mode 100644 index 00000000..ce4349bc --- /dev/null +++ b/test/bdd/features/osm2pgsql/import/interpolation.feature @@ -0,0 +1,44 @@ +Feature: Import of interpolations + Test if interpolation objects are correctly imported into the + place_interpolation table + + Background: + Given the grid + | 1 | 2 | + | 4 | 3 | + + Scenario: Simple address interpolations + When loading osm data + """ + n1 + n2 + w13001 Taddr:interpolation=odd,addr:street=Blumenstrasse Nn1,n2 + w13002 Taddr:interpolation=even,place=city Nn1,n2 + w13003 Taddr:interpolation=odd Nn1,n1 + """ + Then place contains exactly + | object | class | type | + | W13002 | place | city | + And place_interpolation contains exactly + | object | type | address!dict | nodes!ints | geometry!wkt | + | W13001 | odd | "street": "Blumenstrasse" | 1,2 | 1,2 | + | W13002 | even | - | 1,2 | 1,2 | + + Scenario: Address interpolation with housenumber + When loading osm data + """ + n1 + n2 + n3 + n4 + w34 Taddr:interpolation=all,addr:housenumber=2-4,building=yes Nn1,n2,n3,n4,n1 + w35 Taddr:interpolation=all,addr:housenumber=5,building=yes Nn1,n2,n3,n4,n1 + w36 Taddr:interpolation=all,addr:housenumber=2a-c,building=yes Nn1,n2,n3,n4,n1 + """ + Then place contains exactly + | object | class | type | address!dict | + | w35 | building | yes | "housenumber" : "5", "interpolation": "all" | + | w36 | building | yes | "housenumber" : "2a-c", "interpolation": "all" | + Then place_interpolation contains exactly + | object | type | address!dict | nodes!ints | geometry!wkt | + | W34 | all | "housenumber": "2-4" | - | (1,2,3,4,1) | diff --git a/test/bdd/features/osm2pgsql/import/tags.feature b/test/bdd/features/osm2pgsql/import/tags.feature index b59da688..f27847a4 100644 --- a/test/bdd/features/osm2pgsql/import/tags.feature +++ b/test/bdd/features/osm2pgsql/import/tags.feature @@ -205,24 +205,6 @@ Feature: Tag evaluation | N12005 | 12345 | - | - Scenario: Address interpolations - Given the grid - | 1 | 2 | - When loading osm data - """ - n1 - n2 - w13001 Taddr:interpolation=odd Nn1,n2 - w13002 Taddr:interpolation=even,place=city Nn1,n2 - """ - Then place contains exactly - | object | - And place_interpolation contains exactly - | object | type | address!dict | - | W13001 | odd | - | - | W13002 | even | - | - - Scenario: Footways When loading osm data """ diff --git a/test/bdd/utils/checks.py b/test/bdd/utils/checks.py index 1d09f378..ade29438 100644 --- a/test/bdd/utils/checks.py +++ b/test/bdd/utils/checks.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Helper functions to compare expected values. @@ -61,6 +61,8 @@ COMPARISON_FUNCS = { 'fm': lambda val, exp: re.fullmatch(exp, val) is not None, 'dict': lambda val, exp: (val is None if exp == '-' else (val == ast.literal_eval('{' + exp + '}'))), + 'ints': lambda val, exp: (val is None if exp == '-' + else (val == [int(i) for i in exp.split(',')])), 'in_box': within_box } @@ -84,6 +86,8 @@ class ResultAttr: !fm - consider comparison string a regular expression and match full value !wkt - convert the expected value to a WKT string before comparing !in_box - the expected value is a comma-separated bbox description + !dict - compare as a dictitionary, member order does not matter + !ints - compare as integer array """ def __init__(self, obj, key, grid=None): From abd5cbada6aa22316960e1ec867f1130cc85c1e2 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 18 Feb 2026 20:01:15 +0100 Subject: [PATCH 08/10] revert importing of housenumber interpolations --- lib-lua/themes/nominatim/init.lua | 54 ++- lib-sql/functions/interpolation.sql | 389 ++++++++---------- lib-sql/functions/placex_triggers.sql | 6 +- lib-sql/functions/utils.sql | 4 +- lib-sql/indices.sql | 1 - lib-sql/tables/interpolation.sql | 3 +- src/nominatim_db/tools/database_import.py | 4 +- .../features/db/import/interpolation.feature | 4 +- .../features/db/query/housenumbers.feature | 32 -- .../osm2pgsql/import/interpolation.feature | 18 +- .../osm2pgsql/update/interpolations.feature | 22 +- .../osm2pgsql/update/postcodes.feature | 10 +- test/bdd/test_db.py | 26 +- test/python/conftest.py | 10 +- test/python/tools/test_database_import.py | 3 +- 15 files changed, 242 insertions(+), 344 deletions(-) diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index fd15968b..173b9f25 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -79,12 +79,12 @@ local table_definitions = { } }, place_interpolation = { - ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + ids = { type = 'way', id_column = 'osm_id' }, columns = { { column = 'type', type = 'text', not_null = true }, { column = 'address', type = 'hstore' }, - { column = 'nodes', type = 'text', sql_type = 'bigint[]' }, - { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true }, + { column = 'nodes', type = 'text', sql_type = 'bigint[]', not_null = true }, + { column = 'geometry', type = 'linestring', projection = 'WGS84', not_null = true }, }, indexes = { { column = 'nodes', method = 'gin' } @@ -387,6 +387,7 @@ function Place:grab_address_parts(data) if atype ~= nil then if atype == 'main' then + self.has_name = true self.address[strip_address_prefix(k)] = v count = count + 1 elseif atype == 'extra' then @@ -707,38 +708,31 @@ function module.process_tags(o) local fallback = o:grab_name_parts{groups=NAME_FILTER} -- address keys - local fallback_address = (o:grab_address_parts{groups=ADDRESS_FILTER} > 0) + if o:grab_address_parts{groups=ADDRESS_FILTER} > 0 and fallback == nil then + fallback = {'place', 'house', address_fallback} + end if o.address.country ~= nil and #o.address.country ~= 2 then o.address['country'] = nil end - if o.address.interpolation ~= nil - and (o.address.housenumber == nil - or string.find(o.address.housenumber, '^%d+%-%d+$') ~= nil) then - if o:geometry_is_valid() then - local extra_addr = nil - for k, v in pairs(o.address) do - if k ~= 'interpolation' then - if extra_addr == nil then - extra_addr = {} - end - extra_addr[k] = v + if o.address.interpolation ~= nil and o.address.housenumber == nil + and o.object.type == 'way' and o.object.nodes ~= nil then + local extra_addr = nil + for k, v in pairs(o.address) do + if k ~= 'interpolation' then + if extra_addr == nil then + extra_addr = {} end + extra_addr[k] = v end - local nodes = nil - if o.address.housenumber == nil and o.object.nodes ~= nil then - nodes = '{' .. table.concat(o.object.nodes, ',') .. '}' - end - insert_row.place_interpolation{ - type = o.address.interpolation, - address = extra_addr, - nodes = nodes, - geometry = o.geometry - } - fallback_address = false - else - return end + + insert_row.place_interpolation{ + type = o.address.interpolation, + address = extra_addr, + nodes = '{' .. table.concat(o.object.nodes, ',') .. '}', + geometry = o.object:as_linestring() + } end -- collect main keys @@ -760,7 +754,7 @@ function module.process_tags(o) geometry = o.geometry } end - elseif ktype == 'fallback' and (fallback_address or o.has_name) then + elseif ktype == 'fallback' and o.has_name then fallback = {k, v, PlaceTransform.always} end end @@ -769,8 +763,6 @@ function module.process_tags(o) if o.num_entries == 0 then if fallback ~= nil then o:write_place(fallback[1], fallback[2], fallback[3]) - elseif fallback_address then - o:write_place('place', 'house', address_fallback) elseif POSTCODE_FALLBACK and not postcode_collect and o.address.postcode ~= nil and o:geometry_is_valid() then diff --git a/lib-sql/functions/interpolation.sql b/lib-sql/functions/interpolation.sql index d558378d..8a9fc2df 100644 --- a/lib-sql/functions/interpolation.sql +++ b/lib-sql/functions/interpolation.sql @@ -15,67 +15,63 @@ DECLARE existingplacex BIGINT[]; BEGIN - -- Remove the place from the list of places to be deleted - DELETE FROM place_interpolation_to_be_deleted pdel - WHERE pdel.osm_type = NEW.osm_type and pdel.osm_id = NEW.osm_id; - - SELECT * INTO existing FROM place_interpolation p - WHERE p.osm_type = NEW.osm_type AND p.osm_id = NEW.osm_id; - IF NOT (NEW.type in ('odd', 'even', 'all') OR NEW.type similar to '[1-9]') THEN -- the new interpolation is illegal, simply remove existing entries - DELETE FROM location_property_osmline o - WHERE o.osm_type = NEW.osm_type AND o.osm_id = NEW.osm_id; + DELETE FROM location_property_osmline o WHERE o.osm_id = NEW.osm_id; + RETURN NULL; + END IF; + + -- Remove the place from the list of places to be deleted + DELETE FROM place_interpolation_to_be_deleted pdel WHERE pdel.osm_id = NEW.osm_id; + + SELECT * INTO existing FROM place_interpolation p WHERE p.osm_id = NEW.osm_id; + + -- Get the existing entry from the interpolation table. + SELECT array_agg(place_id) INTO existingplacex + FROM location_property_osmline o WHERE o.osm_id = NEW.osm_id; + + IF array_length(existingplacex, 1) is NULL THEN + INSERT INTO location_property_osmline (osm_id, type, address, linegeo) + VALUES (NEW.osm_id, NEW.type, NEW.address, NEW.geometry); ELSE - -- Get the existing entry from the interpolation table. - SELECT array_agg(place_id) INTO existingplacex - FROM location_property_osmline o - WHERE o.osm_type = NEW.osm_type AND o.osm_id = NEW.osm_id; - - IF array_length(existingplacex, 1) is NULL THEN - INSERT INTO location_property_osmline (osm_type, osm_id, type, address, linegeo) - VALUES (NEW.osm_type, NEW.osm_id, NEW.type, NEW.address, NEW.geometry); - ELSE - -- Update the interpolation table: - -- The first entry gets the original data, all other entries - -- are removed and will be recreated on indexing. - -- (An interpolation can be split up, if it has more than 2 address nodes) - -- Update unconditionally here as the changes might be coming from the - -- nodes on the interpolation. - UPDATE location_property_osmline - SET type = NEW.type, - address = NEW.address, - linegeo = NEW.geometry, - startnumber = null, - indexed_status = 1 - WHERE place_id = existingplacex[1]; - IF array_length(existingplacex, 1) > 1 THEN - DELETE FROM location_property_osmline - WHERE place_id = any(existingplacex[2:]); - END IF; + -- Update the interpolation table: + -- The first entry gets the original data, all other entries + -- are removed and will be recreated on indexing. + -- (An interpolation can be split up, if it has more than 2 address nodes) + -- Update unconditionally here as the changes might be coming from the + -- nodes on the interpolation. + UPDATE location_property_osmline + SET type = NEW.type, + address = NEW.address, + linegeo = NEW.geometry, + startnumber = null, + indexed_status = 1 + WHERE place_id = existingplacex[1]; + IF array_length(existingplacex, 1) > 1 THEN + DELETE FROM location_property_osmline WHERE place_id = any(existingplacex[2:]); END IF; + END IF; - -- need to invalidate nodes because they might copy address info - IF NEW.address is not NULL - AND (existing.osm_type is NULL - OR coalesce(existing.address, ''::hstore) != NEW.address) - THEN - UPDATE placex SET indexed_status = 2 - WHERE osm_type = 'N' AND osm_id = ANY(NEW.nodes) - AND indexed_status = 0; - END IF; + -- need to invalidate nodes because they might copy address info + IF NEW.address is not NULL + AND (existing.osm_id is NULL + OR coalesce(existing.address, ''::hstore) != NEW.address) + THEN + UPDATE placex SET indexed_status = 2 + WHERE osm_type = 'N' AND osm_id = ANY(NEW.nodes) AND indexed_status = 0; END IF; -- finally update/insert place_interpolation itself - IF existing.osm_type is not NULL THEN + IF existing.osm_id is not NULL THEN -- Always updates as the nodes with the housenumber might be the reason -- for the change. UPDATE place_interpolation p SET type = NEW.type, address = NEW.address, + nodes = NEW.nodes, geometry = NEW.geometry - WHERE p.osm_type = NEW.osm_type AND p.osm_id = NEW.osm_id; + WHERE p.osm_id = NEW.osm_id; RETURN NULL; END IF; @@ -91,15 +87,15 @@ CREATE OR REPLACE FUNCTION place_interpolation_delete() DECLARE deferred BOOLEAN; BEGIN - {% if debug %}RAISE WARNING 'Delete for % % %/%', OLD.osm_type, OLD.osm_id, OLD.class, OLD.type;{% endif %} + {% if debug %}RAISE WARNING 'Delete for interpolation %', OLD.osm_id;{% endif %} - INSERT INTO place_interpolation_to_be_deleted (osm_type, osm_id) - VALUES(OLD.osm_type, OLD.osm_id); + INSERT INTO place_interpolation_to_be_deleted (osm_id) VALUES(OLD.osm_id); RETURN NULL; END; $$ LANGUAGE plpgsql; + CREATE OR REPLACE FUNCTION get_interpolation_address(in_address HSTORE, wayid BIGINT) RETURNS HSTORE AS $$ @@ -111,8 +107,7 @@ BEGIN RETURN in_address; END IF; - SELECT nodes INTO waynodes FROM place_interpolation - WHERE osm_type = 'W' AND osm_id = wayid; + SELECT nodes INTO waynodes FROM place_interpolation WHERE osm_id = wayid; IF array_upper(waynodes, 1) IS NOT NULL THEN FOR location IN @@ -184,22 +179,7 @@ BEGIN RETURN NULL; END IF; - IF NEW.address is not NULL AND NEW.address ? 'housenumber' THEN - IF NEW.address->'housenumber' not similar to '[0-9]+-[0-9]+' THEN - -- housenumber needs to look like an interpolation - RETURN NULL; - END IF; - - centroid := ST_Centroid(NEW.linegeo); - -- interpolation of a housenumber, make sure we have a line to interpolate on - NEW.geometry := ST_MakeLine(centroid, ST_Project(centroid, 0.0000001, 0)); - ELSE - centroid := get_center_point(NEW.linegeo); - IF NEW.osm_type != 'W' THEN - RETURN NULL; - END IF; - END IF; - + centroid := get_center_point(NEW.linegeo); NEW.indexed_status := 1; --STATUS_NEW NEW.country_code := lower(get_country_code(centroid)); @@ -230,7 +210,6 @@ DECLARE sectiongeo GEOMETRY; postcode TEXT; stepmod SMALLINT; - splitstring TEXT[]; BEGIN -- deferred delete IF OLD.indexed_status = 100 THEN @@ -261,158 +240,132 @@ BEGIN stepmod := NULL; END IF; - IF NEW.address is not NULL AND NEW.address ? 'housenumer' THEN - -- interpolation interval is in housenumber - splitstring := string_to_array(NEW.address->'housenumber', '-'); - NEW.startnumber := (splitstring[1])::INTEGER; - NEW.endnumber := (splitstring[2])::INTEGER; + SELECT nodes INTO waynodes FROM place_interpolation WHERE osm_id = NEW.osm_id; - IF stepmod is not NULL THEN - IF NEW.startnumber % NEW.step != stepmod THEN - NEW.startnumber := NEW.startnumber + 1; - END IF; - IF NEW.endnumber % NEW.step != stepmod THEN - NEW.endnumber := NEW.endnumber - 1; - END IF; - ELSE - NEW.endnumber := NEW.startnumber + ((NEW.endnumber - NEW.startnumber) / NEW.step) * NEW.step; - END IF; - - IF NEW.startnumber = NEW.endnumber THEN - NEW.geometry := ST_PointN(NEW.geometry, 1); - ELSEIF NEW.startnumber > NEW.endnumber THEN - NEW.startnumber := NULL; - END IF; - ELSE - -- classic interpolation way - SELECT nodes INTO waynodes - FROM place_interpolation WHERE osm_type = NEW.osm_type AND osm_id = NEW.osm_id; - - IF array_upper(waynodes, 1) IS NULL THEN - RETURN NEW; - END IF; - - linegeo := null; - SELECT null::integer as hnr INTO prevnode; - - -- Go through all nodes on the interpolation line that have a housenumber. - FOR nextnode IN - SELECT DISTINCT ON (nodeidpos) - osm_id, address, geometry, - -- Take the postcode from the node only if it has a housenumber itself. - -- Note that there is a corner-case where the node has a wrongly - -- formatted postcode and therefore 'postcode' contains a derived - -- variant. - CASE WHEN address ? 'postcode' THEN placex.postcode ELSE NULL::text END as postcode, - (address->'housenumber')::integer as hnr - FROM placex, generate_series(1, array_upper(waynodes, 1)) nodeidpos - WHERE osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT - and address is not NULL and address ? 'housenumber' - and address->'housenumber' ~ '^[0-9]{1,6}$' - and ST_Distance(NEW.linegeo, geometry) < 0.0005 - ORDER BY nodeidpos - LOOP - {% if debug %}RAISE WARNING 'processing point % (%)', nextnode.hnr, ST_AsText(nextnode.geometry);{% endif %} - IF linegeo is null THEN - linegeo := NEW.linegeo; - ELSE - splitpoint := ST_LineLocatePoint(linegeo, nextnode.geometry); - IF splitpoint = 0 THEN - -- Corner case where the splitpoint falls on the first point - -- and thus would not return a geometry. Skip that section. - sectiongeo := NULL; - ELSEIF splitpoint = 1 THEN - -- Point is at the end of the line. - sectiongeo := linegeo; - linegeo := NULL; - ELSE - -- Split the line. - sectiongeo := ST_LineSubstring(linegeo, 0, splitpoint); - linegeo := ST_LineSubstring(linegeo, splitpoint, 1); - END IF; - END IF; - - IF prevnode.hnr is not null - -- Check if there are housenumbers to interpolate between the - -- regularly mapped housenumbers. - -- (Conveniently also fails if one of the house numbers is not a number.) - and abs(prevnode.hnr - nextnode.hnr) > NEW.step - -- If the interpolation geometry is broken or two nodes are at the - -- same place, then splitting might produce a point. Ignore that. - and ST_GeometryType(sectiongeo) = 'ST_LineString' - THEN - IF prevnode.hnr < nextnode.hnr THEN - startnumber := prevnode.hnr; - endnumber := nextnode.hnr; - ELSE - startnumber := nextnode.hnr; - endnumber := prevnode.hnr; - sectiongeo := ST_Reverse(sectiongeo); - END IF; - - -- Adjust the interpolation, so that only inner housenumbers - -- are taken into account. - IF stepmod is null THEN - newstart := startnumber + NEW.step; - ELSE - newstart := startnumber + 1; - moddiff := newstart % NEW.step - stepmod; - IF moddiff < 0 THEN - newstart := newstart + (NEW.step + moddiff); - ELSE - newstart := newstart + moddiff; - END IF; - END IF; - newend := newstart + ((endnumber - 1 - newstart) / NEW.step) * NEW.step; - - -- If newstart and newend are the same, then this returns a point. - sectiongeo := ST_LineSubstring(sectiongeo, - (newstart - startnumber)::float / (endnumber - startnumber)::float, - (newend - startnumber)::float / (endnumber - startnumber)::float); - startnumber := newstart; - endnumber := newend; - - -- determine postcode - postcode := coalesce(prevnode.postcode, nextnode.postcode, postcode); - IF postcode is NULL and NEW.parent_place_id > 0 THEN - SELECT placex.postcode FROM placex - WHERE place_id = NEW.parent_place_id INTO postcode; - END IF; - IF postcode is NULL THEN - postcode := get_nearest_postcode(NEW.country_code, nextnode.geometry); - END IF; - - -- Add the interpolation. If this is the first segment, just modify - -- the interpolation to be inserted, otherwise add an additional one - -- (marking it indexed already). - IF NEW.startnumber IS NULL THEN - NEW.startnumber := startnumber; - NEW.endnumber := endnumber; - NEW.linegeo := ST_ReducePrecision(sectiongeo, 0.0000001); - NEW.postcode := postcode; - ELSE - INSERT INTO location_property_osmline - (linegeo, partition, osm_type, osm_id, parent_place_id, - startnumber, endnumber, step, type, - address, postcode, country_code, - geometry_sector, indexed_status) - VALUES (ST_ReducePrecision(sectiongeo, 0.0000001), - NEW.partition, NEW.osm_type, NEW.osm_id, NEW.parent_place_id, - startnumber, endnumber, NEW.step, NEW.type, - NEW.address, postcode, - NEW.country_code, NEW.geometry_sector, 0); - END IF; - END IF; - - -- early break if we are out of line string, - -- might happen when a line string loops back on itself - IF linegeo is null or ST_GeometryType(linegeo) != 'ST_LineString' THEN - RETURN NEW; - END IF; - - prevnode := nextnode; - END LOOP; + IF array_upper(waynodes, 1) IS NULL THEN + RETURN NEW; END IF; + + linegeo := null; + SELECT null::integer as hnr INTO prevnode; + + -- Go through all nodes on the interpolation line that have a housenumber. + FOR nextnode IN + SELECT DISTINCT ON (nodeidpos) + osm_id, address, geometry, + -- Take the postcode from the node only if it has a housenumber itself. + -- Note that there is a corner-case where the node has a wrongly + -- formatted postcode and therefore 'postcode' contains a derived + -- variant. + CASE WHEN address ? 'postcode' THEN placex.postcode ELSE NULL::text END as postcode, + (address->'housenumber')::integer as hnr + FROM placex, generate_series(1, array_upper(waynodes, 1)) nodeidpos + WHERE osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT + and address is not NULL and address ? 'housenumber' + and address->'housenumber' ~ '^[0-9]{1,6}$' + and ST_Distance(NEW.linegeo, geometry) < 0.0005 + ORDER BY nodeidpos + LOOP + {% if debug %}RAISE WARNING 'processing point % (%)', nextnode.hnr, ST_AsText(nextnode.geometry);{% endif %} + IF linegeo is null THEN + linegeo := NEW.linegeo; + ELSE + splitpoint := ST_LineLocatePoint(linegeo, nextnode.geometry); + IF splitpoint = 0 THEN + -- Corner case where the splitpoint falls on the first point + -- and thus would not return a geometry. Skip that section. + sectiongeo := NULL; + ELSEIF splitpoint = 1 THEN + -- Point is at the end of the line. + sectiongeo := linegeo; + linegeo := NULL; + ELSE + -- Split the line. + sectiongeo := ST_LineSubstring(linegeo, 0, splitpoint); + linegeo := ST_LineSubstring(linegeo, splitpoint, 1); + END IF; + END IF; + + IF prevnode.hnr is not null + -- Check if there are housenumbers to interpolate between the + -- regularly mapped housenumbers. + -- (Conveniently also fails if one of the house numbers is not a number.) + and abs(prevnode.hnr - nextnode.hnr) > NEW.step + -- If the interpolation geometry is broken or two nodes are at the + -- same place, then splitting might produce a point. Ignore that. + and ST_GeometryType(sectiongeo) = 'ST_LineString' + THEN + IF prevnode.hnr < nextnode.hnr THEN + startnumber := prevnode.hnr; + endnumber := nextnode.hnr; + ELSE + startnumber := nextnode.hnr; + endnumber := prevnode.hnr; + sectiongeo := ST_Reverse(sectiongeo); + END IF; + + -- Adjust the interpolation, so that only inner housenumbers + -- are taken into account. + IF stepmod is null THEN + newstart := startnumber + NEW.step; + ELSE + newstart := startnumber + 1; + moddiff := newstart % NEW.step - stepmod; + IF moddiff < 0 THEN + newstart := newstart + (NEW.step + moddiff); + ELSE + newstart := newstart + moddiff; + END IF; + END IF; + newend := newstart + ((endnumber - 1 - newstart) / NEW.step) * NEW.step; + + -- If newstart and newend are the same, then this returns a point. + sectiongeo := ST_LineSubstring(sectiongeo, + (newstart - startnumber)::float / (endnumber - startnumber)::float, + (newend - startnumber)::float / (endnumber - startnumber)::float); + startnumber := newstart; + endnumber := newend; + + -- determine postcode + postcode := coalesce(prevnode.postcode, nextnode.postcode, postcode); + IF postcode is NULL and NEW.parent_place_id > 0 THEN + SELECT placex.postcode FROM placex + WHERE place_id = NEW.parent_place_id INTO postcode; + END IF; + IF postcode is NULL THEN + postcode := get_nearest_postcode(NEW.country_code, nextnode.geometry); + END IF; + + -- Add the interpolation. If this is the first segment, just modify + -- the interpolation to be inserted, otherwise add an additional one + -- (marking it indexed already). + IF NEW.startnumber IS NULL THEN + NEW.startnumber := startnumber; + NEW.endnumber := endnumber; + NEW.linegeo := ST_ReducePrecision(sectiongeo, 0.0000001); + NEW.postcode := postcode; + ELSE + INSERT INTO location_property_osmline + (linegeo, partition, osm_id, parent_place_id, + startnumber, endnumber, step, type, + address, postcode, country_code, + geometry_sector, indexed_status) + VALUES (ST_ReducePrecision(sectiongeo, 0.0000001), + NEW.partition, NEW.osm_id, NEW.parent_place_id, + startnumber, endnumber, NEW.step, NEW.type, + NEW.address, postcode, + NEW.country_code, NEW.geometry_sector, 0); + END IF; + END IF; + + -- early break if we are out of line string, + -- might happen when a line string loops back on itself + IF linegeo is null or ST_GeometryType(linegeo) != 'ST_LineString' THEN + RETURN NEW; + END IF; + + prevnode := nextnode; + END LOOP; END IF; RETURN NEW; diff --git a/lib-sql/functions/placex_triggers.sql b/lib-sql/functions/placex_triggers.sql index 00868589..19764ed9 100644 --- a/lib-sql/functions/placex_triggers.sql +++ b/lib-sql/functions/placex_triggers.sql @@ -53,10 +53,8 @@ BEGIN -- See if we can inherit additional address tags from an interpolation. -- These will become permanent. FOR location IN - SELECT address as address - FROM place_interpolation - WHERE p.osm_id = any(place_interpolation.nodes) - AND address is not NULL AND not address ? 'housenumber' + SELECT address FROM place_interpolation + WHERE ARRAY[p.osm_id] && place_interpolation.nodes AND address is not NULL LOOP result.address := location.address || result.address; END LOOP; diff --git a/lib-sql/functions/utils.sql b/lib-sql/functions/utils.sql index 878301a0..f1788bcb 100644 --- a/lib-sql/functions/utils.sql +++ b/lib-sql/functions/utils.sql @@ -630,12 +630,12 @@ BEGIN -- delete from place_interpolation table ALTER TABLE place_interpolation DISABLE TRIGGER place_interpolation_before_delete; DELETE FROM place_interpolation p USING place_interpolation_to_be_deleted d - WHERE p.osm_type = d.osm_type AND p.osm_id = d.osm_id; + WHERE p.osm_id = d.osm_id; ALTER TABLE place_interpolation ENABLE TRIGGER place_interpolation_before_delete; UPDATE location_property_osmline o SET indexed_status = 100 FROM place_interpolation_to_be_deleted d - WHERE o.osm_type = d.osm_Type AND o.osm_id = d.osm_id; + WHERE o.osm_id = d.osm_id; TRUNCATE TABLE place_interpolation_to_be_deleted; diff --git a/lib-sql/indices.sql b/lib-sql/indices.sql index f3b79705..f350f951 100644 --- a/lib-sql/indices.sql +++ b/lib-sql/indices.sql @@ -75,7 +75,6 @@ CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id ); CREATE TABLE IF NOT EXISTS place_interpolation_to_be_deleted ( - osm_type CHAR(1) NOT NULL, osm_id BIGINT NOT NULL ); --- diff --git a/lib-sql/tables/interpolation.sql b/lib-sql/tables/interpolation.sql index a7b53209..1a4d72af 100644 --- a/lib-sql/tables/interpolation.sql +++ b/lib-sql/tables/interpolation.sql @@ -8,7 +8,6 @@ DROP TABLE IF EXISTS location_property_osmline; CREATE TABLE location_property_osmline ( place_id BIGINT NOT NULL, - osm_type CHAR(1) NOT NULL, osm_id BIGINT NOT NULL, parent_place_id BIGINT, geometry_sector INTEGER NOT NULL, @@ -19,7 +18,7 @@ CREATE TABLE location_property_osmline ( step SMALLINT, partition SMALLINT NOT NULL, indexed_status SMALLINT NOT NULL, - linegeo GEOMETRY NOT NULL, + linegeo GEOMETRY(Geometry, 4326) NOT NULL, address HSTORE, token_info JSONB, -- custom column for tokenizer use only postcode TEXT, diff --git a/src/nominatim_db/tools/database_import.py b/src/nominatim_db/tools/database_import.py index 388cca0f..4bf9c406 100644 --- a/src/nominatim_db/tools/database_import.py +++ b/src/nominatim_db/tools/database_import.py @@ -225,8 +225,8 @@ async def load_data(dsn: str, threads: int) -> None: # Interpolations need to be copied separately await pool.put_query(""" - INSERT INTO location_property_osmline (osm_type, osm_id, type, address, linegeo) - SELECT osm_type, osm_id, type, address, geometry + INSERT INTO location_property_osmline (osm_id, type, address, linegeo) + SELECT osm_id, type, address, geometry FROM place_interpolation """, None) diff --git a/test/bdd/features/db/import/interpolation.feature b/test/bdd/features/db/import/interpolation.feature index cc332852..764a062e 100644 --- a/test/bdd/features/db/import/interpolation.feature +++ b/test/bdd/features/db/import/interpolation.feature @@ -378,8 +378,8 @@ Feature: Import of address interpolations Scenario: Ways without node entries are ignored Given the interpolations - | osm | type | geometry | - | W1 | even | 1 1, 1 1.001 | + | osm | type | geometry | nodes | + | W1 | even | 1 1, 1 1.001 | 34,45 | And the named places | osm | class | type | geometry | | W10 | highway | residential | 1 1, 1 1.001 | diff --git a/test/bdd/features/db/query/housenumbers.feature b/test/bdd/features/db/query/housenumbers.feature index df466ecb..8e1b1e5d 100644 --- a/test/bdd/features/db/query/housenumbers.feature +++ b/test/bdd/features/db/query/housenumbers.feature @@ -287,38 +287,6 @@ Feature: Searching of house numbers | N1 | - Scenario: Interpolations are found according to their type - Given the grid - | 10 | | 11 | - | 100 | | 101 | - | 20 | | 21 | - And the places - | osm | class | type | name | geometry | - | W100 | highway | residential | Ringstr | 100, 101 | - And the places - | osm | class | type | addr+interpolation | geometry | - | W10 | place | houses | even | 10, 11 | - | W20 | place | houses | odd | 20, 21 | - And the places - | osm | class | type | housenr | geometry | - | N10 | place | house | 10 | 10 | - | N11 | place | house | 20 | 11 | - | N20 | place | house | 11 | 20 | - | N21 | place | house | 21 | 21 | - And the ways - | id | nodes | - | 10 | 10, 11 | - | 20 | 20, 21 | - When importing - When geocoding "Ringstr 12" - Then the result set contains - | object | - | W10 | - When geocoding "Ringstr 13" - Then the result set contains - | object | - | W20 | - Scenario: A housenumber with interpolation is found Given the places | osm | class | type | housenr | addr+interpolation | geometry | diff --git a/test/bdd/features/osm2pgsql/import/interpolation.feature b/test/bdd/features/osm2pgsql/import/interpolation.feature index ce4349bc..07e53685 100644 --- a/test/bdd/features/osm2pgsql/import/interpolation.feature +++ b/test/bdd/features/osm2pgsql/import/interpolation.feature @@ -20,9 +20,9 @@ Feature: Import of interpolations | object | class | type | | W13002 | place | city | And place_interpolation contains exactly - | object | type | address!dict | nodes!ints | geometry!wkt | - | W13001 | odd | "street": "Blumenstrasse" | 1,2 | 1,2 | - | W13002 | even | - | 1,2 | 1,2 | + | osm_id | type | address!dict | nodes!ints | geometry!wkt | + | 13001 | odd | "street": "Blumenstrasse" | 1,2 | 1,2 | + | 13002 | even | - | 1,2 | 1,2 | Scenario: Address interpolation with housenumber When loading osm data @@ -33,12 +33,10 @@ Feature: Import of interpolations n4 w34 Taddr:interpolation=all,addr:housenumber=2-4,building=yes Nn1,n2,n3,n4,n1 w35 Taddr:interpolation=all,addr:housenumber=5,building=yes Nn1,n2,n3,n4,n1 - w36 Taddr:interpolation=all,addr:housenumber=2a-c,building=yes Nn1,n2,n3,n4,n1 + w36 Taddr:interpolation=all,addr:housenumber=2a-c Nn1,n2,n3,n4,n1 """ Then place contains exactly - | object | class | type | address!dict | - | w35 | building | yes | "housenumber" : "5", "interpolation": "all" | - | w36 | building | yes | "housenumber" : "2a-c", "interpolation": "all" | - Then place_interpolation contains exactly - | object | type | address!dict | nodes!ints | geometry!wkt | - | W34 | all | "housenumber": "2-4" | - | (1,2,3,4,1) | + | object | class | type | address!dict | + | W35 | building | yes | "housenumber": "5", "interpolation": "all" | + | W34 | building | yes | "housenumber": "2-4", "interpolation": "all" | + | W36 | place | house | "housenumber": "2a-c", "interpolation": "all" | diff --git a/test/bdd/features/osm2pgsql/update/interpolations.feature b/test/bdd/features/osm2pgsql/update/interpolations.feature index 6c76c91b..b0936d2c 100644 --- a/test/bdd/features/osm2pgsql/update/interpolations.feature +++ b/test/bdd/features/osm2pgsql/update/interpolations.feature @@ -30,8 +30,8 @@ Feature: Updates of address interpolation objects | N2 | place | house | | W33 | highway | residential | And place_interpolation contains exactly - | object | type | - | W99 | odd | + | osm_id | type | + | 99 | odd | When indexing Then placex contains exactly | object | class | type | @@ -55,8 +55,8 @@ Feature: Updates of address interpolation objects | N1 | place | house | | N2 | place | house | And place_interpolation contains exactly - | object | type | - | W99 | odd | + | osm_id | type | + | 99 | odd | When updating osm data """ @@ -67,7 +67,7 @@ Feature: Updates of address interpolation objects | N1 | place | house | | N2 | place | house | And place_interpolation contains exactly - | object | + | osm_id | When indexing Then placex contains exactly | object | class | type | @@ -92,7 +92,7 @@ Feature: Updates of address interpolation objects | W33 | highway | residential | | W99 | highway | residential | And place_interpolation contains exactly - | object | + | osm_id | When updating osm data """ @@ -104,8 +104,8 @@ Feature: Updates of address interpolation objects | N2 | place | house | | W33 | highway | residential | And place_interpolation contains exactly - | object | type | - | W99 | odd | + | osm_id | type | + | 99 | odd | When indexing Then placex contains exactly | object | class | type | @@ -129,8 +129,8 @@ Feature: Updates of address interpolation objects | N1 | place | house | | N2 | place | house | And place_interpolation contains exactly - | object | type | - | W99 | odd | + | osm_id | type | + | 99 | odd | When updating osm data """ @@ -142,7 +142,7 @@ Feature: Updates of address interpolation objects | N2 | place | house | | W99 | highway | residential | And place_interpolation contains exactly - | object | + | osm_id | When indexing Then placex contains exactly | object | class | type | diff --git a/test/bdd/features/osm2pgsql/update/postcodes.feature b/test/bdd/features/osm2pgsql/update/postcodes.feature index 3ae73f27..dd462d1c 100644 --- a/test/bdd/features/osm2pgsql/update/postcodes.feature +++ b/test/bdd/features/osm2pgsql/update/postcodes.feature @@ -113,8 +113,8 @@ Feature: Update of postcode only objects | N1 | place | house | | N2 | place | house | And place_interpolation contains exactly - | object | type | - | W34 | odd | + | osm_id | type | + | 34 | odd | When updating osm data """ @@ -128,7 +128,7 @@ Feature: Update of postcode only objects | object | postcode | | W34 | 4456 | And place_interpolation contains exactly - | object | + | osm_id | When indexing Then location_property_osmline contains exactly | osm_id | @@ -163,8 +163,8 @@ Feature: Update of postcode only objects | N2 | place | house | | W33 | highway | residential | And place_interpolation contains exactly - | object | type | - | W34 | odd | + | osm_id | type | + | 34 | odd | And place_postcode contains exactly | object | When indexing diff --git a/test/bdd/test_db.py b/test/bdd/test_db.py index 91b5b922..47a99578 100644 --- a/test/bdd/test_db.py +++ b/test/bdd/test_db.py @@ -135,15 +135,12 @@ def import_place_interpolations(row_factory, datatable, node_grid): """ for row in datatable[1:]: data = PlaceColumn(node_grid).add_row(datatable[0], row, False) + assert data.columns['osm_type'] == 'W' - if 'nodes' in data.columns: - nodes = [int(x) for x in data.columns['nodes'].split(',')] - else: - nodes = None - - params = {'osm_type': data.columns['osm_type'], 'osm_id': data.columns['osm_id'], + params = {'osm_id': data.columns['osm_id'], 'type': data.columns['type'], - 'address': data.columns.get('address'), 'nodes': nodes, + 'address': data.columns.get('address'), + 'nodes': [int(x) for x in data.columns['nodes'].split(',')], 'geometry': pysql.SQL(data.get_wkt())} row_factory('place_interpolation', **params) @@ -275,15 +272,12 @@ def update_place_interpolations(db_conn, row_factory, update_config, datatable, """ for row in datatable[1:]: data = PlaceColumn(node_grid).add_row(datatable[0], row, False) + assert data.columns['osm_type'] == 'W' - if 'nodes' in data.columns: - nodes = [int(x) for x in data.columns['nodes'].split(',')] - else: - nodes = None - - params = {'osm_type': data.columns['osm_type'], 'osm_id': data.columns['osm_id'], + params = {'osm_id': data.columns['osm_id'], 'type': data.columns['type'], - 'address': data.columns.get('address'), 'nodes': nodes, + 'address': data.columns.get('address'), + 'nodes': [int(x) for x in data.columns['nodes'].split(',')], 'geometry': pysql.SQL(data.get_wkt())} row_factory('place_interpolation', **params) @@ -310,8 +304,8 @@ def do_delete_place(db_conn, update_config, node_grid, otype, oid): cur.execute('TRUNCATE place_to_be_deleted') cur.execute('DELETE FROM place WHERE osm_type = %s and osm_id = %s', (otype, oid)) - cur.execute('DELETE FROM place_interpolation WHERE osm_type = %s and osm_id = %s', - (otype, oid)) + cur.execute('DELETE FROM place_interpolation WHERE osm_id = %s', + (oid, )) cur.execute('SELECT flush_deleted_places()') if otype == 'N': cur.execute('DELETE FROM place_entrance WHERE osm_id = %s', diff --git a/test/python/conftest.py b/test/python/conftest.py index 9617be5a..fbfda1de 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -251,8 +251,7 @@ def place_interpolation_table(temp_db_with_extensions, table_factory): """ Create an empty version of the place_interpolation table. """ table_factory('place_interpolation', - """osm_type char(1) NOT NULL, - osm_id bigint NOT NULL, + """osm_id bigint NOT NULL, type TEXT, address HSTORE, nodes BIGINT[], @@ -266,9 +265,9 @@ def place_interpolation_row(place_interpolation_table, temp_db_cursor): """ idseq = itertools.count(30001) - def _insert(osm_type='N', osm_id=None, typ='odd', address=None, + def _insert(osm_id=None, typ='odd', address=None, nodes=None, geom='LINESTRING(0.1 0.21, 0.1 0.2)'): - params = {'osm_type': osm_type, 'osm_id': osm_id or next(idseq), + params = {'osm_id': osm_id or next(idseq), 'type': typ, 'address': address, 'nodes': nodes, 'geometry': _with_srid(geom)} temp_db_cursor.insert_row('place_interpolation', **params) @@ -319,11 +318,10 @@ def osmline_table(temp_db_with_extensions, load_sql): def osmline_row(osmline_table, temp_db_cursor): idseq = itertools.count(20001) - def _add(osm_type='W', osm_id=None, geom='LINESTRING(12.0 11.0, 12.003 11.0)'): + def _add(osm_id=None, geom='LINESTRING(12.0 11.0, 12.003 11.0)'): return temp_db_cursor.insert_row( 'location_property_osmline', place_id=pysql.SQL("nextval('seq_place')"), - osm_type=osm_type, osm_id=osm_id or next(idseq), geometry_sector=pysql.Literal(20), partition=pysql.Literal(0), diff --git a/test/python/tools/test_database_import.py b/test/python/tools/test_database_import.py index 274fd529..3e98b575 100644 --- a/test/python/tools/test_database_import.py +++ b/test/python/tools/test_database_import.py @@ -169,8 +169,7 @@ async def test_load_data(dsn, place_row, place_interpolation_row, placex_table, temp_db_cursor, threads): for oid in range(100, 130): place_row(osm_id=oid) - place_interpolation_row(osm_type='W', osm_id=342, typ='odd', - geom='LINESTRING(0 0, 10 10)') + place_interpolation_row(osm_id=342, typ='odd', geom='LINESTRING(0 0, 10 10)') temp_db_cursor.execute(""" CREATE OR REPLACE FUNCTION placex_insert() RETURNS TRIGGER AS $$ From d759b6ed00f831e0100ca6e981d17b2b87835387 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 19 Feb 2026 11:07:43 +0100 Subject: [PATCH 09/10] add migration for interpolation tables --- src/nominatim_db/tools/migration.py | 72 ++++++++++++++++++++++++++++- src/nominatim_db/version.py | 4 +- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/nominatim_db/tools/migration.py b/src/nominatim_db/tools/migration.py index 017ba31e..7d8e8ce9 100644 --- a/src/nominatim_db/tools/migration.py +++ b/src/nominatim_db/tools/migration.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Functions for database migration to newer software versions. @@ -350,3 +350,73 @@ def create_place_postcode_table(conn: Connection, config: Configuration, **_: An WHERE osm_type = 'N' and rank_search < 26 and class = 'place'; ANALYSE; """) + + +@_migration(5, 2, 99, 3) +def create_place_interpolation_table(conn: Connection, config: Configuration, **_: Any) -> None: + """ Create place_interpolation table + """ + sqlp = SQLPreprocessor(conn, config) + mutable = not is_frozen(conn) + has_place_table = table_exists(conn, 'place_interpolation') + + if mutable and not has_place_table: + # create tables + conn.execute(""" + CREATE TABLE place_interpolation ( + osm_id BIGINT NOT NULL, + type TEXT NOT NULL, + address HSTORE, + nodes BIGINT[] NOT NULL, + geometry GEOMETRY(LineString, 4326) + ); + + CREATE TABLE IF NOT EXISTS place_interpolation_to_be_deleted ( + osm_id BIGINT NOT NULL + ); + """) + # copy data over + conn.execute(""" + ALTER TABLE place DISABLE TRIGGER ALL; + + WITH deleted AS ( + DELETE FROM place + WHERE class='place' and type = 'houses' + RETURNING osm_type, osm_id, + address->'interpolation' as itype, + address - 'interpolation'::TEXT as address, + geometry) + INSERT INTO place_interpolation (osm_id, type, address, nodes, geometry) + (SELECT d.osm_id, d.itype, d.address, p.nodes, d.geometry + FROM deleted d, planet_osm_ways p + WHERE osm_type = 'W' + AND d.osm_id = p.id + AND itype is not null + AND ST_GeometryType(geometry) = 'ST_LineString'); + + ALTER TABLE place ENABLE TRIGGER ALL; + """) + + # create indices + conn.execute(""" + CREATE INDEX place_interpolation_nodes_idx ON place_interpolation + USING gin(nodes); + CREATE INDEX place_interpolation_osm_id_idx ON place_interpolation + USING btree(osm_id); + """) + # create triggers + sqlp.run_sql_file(conn, 'functions/interpolation.sql') + conn.execute(""" + CREATE TRIGGER place_interpolation_before_insert BEFORE INSERT ON place_interpolation + FOR EACH ROW EXECUTE PROCEDURE place_interpolation_insert(); + CREATE TRIGGER place_interpolation_before_delete BEFORE DELETE ON place_interpolation + FOR EACH ROW EXECUTE PROCEDURE place_interpolation_delete(); + """) + # mutate location_property_osmline table + conn.execute(""" + ALTER TABLE location_property_osmline ADD COLUMN type TEXT; + + UPDATE location_property_osmline + SET type = coalesce(address->'interpolation', 'all'), + address = address - 'interpolation'::TEXT; + """) diff --git a/src/nominatim_db/version.py b/src/nominatim_db/version.py index aab046d2..761803c1 100644 --- a/src/nominatim_db/version.py +++ b/src/nominatim_db/version.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Version information for Nominatim. @@ -55,7 +55,7 @@ def parse_version(version: str) -> NominatimVersion: return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')]) -NOMINATIM_VERSION = parse_version('5.2.99-2') +NOMINATIM_VERSION = parse_version('5.2.99-3') POSTGRESQL_REQUIRED_VERSION = (12, 0) POSTGIS_REQUIRED_VERSION = (3, 0) From 639f05feccff70ef10f64d48093eefcbacebcb3b Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 19 Feb 2026 16:04:12 +0100 Subject: [PATCH 10/10] docs: update database layout in developer section --- docs/develop/Database-Layout.md | 59 +++++++++++++++++++++----- docs/develop/osm2pgsql-tables.plantuml | 21 +++++++++ docs/develop/osm2pgsql-tables.svg | 30 +++++++++++-- docs/develop/search-tables.plantuml | 7 ++- docs/develop/search-tables.svg | 30 +++++++------ 5 files changed, 118 insertions(+), 29 deletions(-) diff --git a/docs/develop/Database-Layout.md b/docs/develop/Database-Layout.md index ad2cdf0b..2bfd9df2 100644 --- a/docs/develop/Database-Layout.md +++ b/docs/develop/Database-Layout.md @@ -11,10 +11,38 @@ The import process creates the following tables: The `planet_osm_*` tables are the usual backing tables for OSM data. Note that Nominatim uses them to look up special relations and to find nodes on -ways. +ways. Apart from those the osm2pgsql import produces three tables as output. -The osm2pgsql import produces a single table `place` as output with the following -columns: +The **place_postcode** table collects postcode information that is not +already present on an object in the place table. That is for one thing +[postcode area relations](https://wiki.openstreetmap.org/wiki/Tag:boundary%3Dpostal_code) +and for another objects with a postcode tag but no other tagging that +qualifies it for inclusion into the geocoding database. + +The table has the following fields: + + * `osm_type` - kind of OSM object (**N** - node, **W** - way, **R** - relation) + * `osm_id` - original OSM ID + * `postcode` - postcode as extacted from the `postcal_code` tag + * `country_code` - computed country code for this postcode. This field + functions as a cache and is only computed when the table is used for + the computation of the final postcodes. + * `centroid` - centroid of the object + * `geometry` - the full geometry of the area for postcode areas only + +The **place_interpolation** table holds all +[address interpolation lines](https://wiki.openstreetmap.org/wiki/Addresses#Interpolation) +and has the following fields: + + * `osm_id` - original OSM ID + * `type` - type of interpolation as extracted from the `addr:interpolation` tag + * `address` - any other `addr:*` tags + * `nodes` - list of OSM nodes contained in this interpolation, + needed to compute the involved housenumbers later + * `geometry` - the linestring for the interpolation (in WSG84) + +The **place** table holds all other OSM object that are interesting and +has the following fields: * `osm_type` - kind of OSM object (**N** - node, **W** - way, **R** - relation) * `osm_id` - original OSM ID @@ -65,23 +93,32 @@ additional columns: * `indexed_status` - processing status of the place (0 - ready, 1 - freshly inserted, 2 - needs updating, 100 - needs deletion) * `indexed_date` - timestamp when the place was processed last * `centroid` - a point feature for the place + * `token_info` - a dummy field used to inject information from the tokenizer + into the indexing process The **location_property_osmline** table is a special table for [address interpolations](https://wiki.openstreetmap.org/wiki/Addresses#Using_interpolation). The columns have the same meaning and use as the columns with the same name in -the placex table. Only three columns are special: +the placex table. Only the following columns are special: - * `startnumber` and `endnumber` - beginning and end of the number range - for the interpolation - * `interpolationtype` - a string `odd`, `even` or `all` to indicate - the interval between the numbers + * `startnumber`, `endnumber` and `step` - beginning and end of the number range + for the interpolation and the increment steps + * `type` - a string to indicate the interval between the numbers as imported + from the OSM `addr:interpolation` tag; valid values are `odd`, `even`, `all` + or a single digit number; interpolations with other values are silently + dropped Address interpolations are always ways in OSM, which is why there is no column `osm_type`. -The **location_postcodes** table holds computed centroids of all postcodes that -can be found in the OSM data. The meaning of the columns is again the same -as that of the placex table. +The **location_postcodes** table holds computed postcode assembled from the +postcode information available in OSM. When a postcode has a postcode area +relation, then the table stores its full geometry. For all other postcode +the centroid is computed using the position of all OSM object that reference +the same postoce. The `osm_id` field can be used to distinguish the two. +When set, it refers to the OSM relation with the postcode area. +The meaning of the columns in the table is again the same as that of the +placex table. Every place needs an address, a set of surrounding places that describe the location of the place. The set of address places is made up of OSM places diff --git a/docs/develop/osm2pgsql-tables.plantuml b/docs/develop/osm2pgsql-tables.plantuml index 89c3c979..8bb7e444 100644 --- a/docs/develop/osm2pgsql-tables.plantuml +++ b/docs/develop/osm2pgsql-tables.plantuml @@ -35,10 +35,31 @@ map place { geometry => GEOMETRY } +map place_postcode { + osm_type => CHAR(1) + osm_id => BIGINT + postcode => TEXT + country_code => TEXT + centroid => GEOMETRY + geometry => GEOMETRY +} + +map place_interpolation { + osm_id => BIGINT + type => TEXT + address => HSTORE + nodes => BIGINT[] + geometry => GEOMETRY +} + + planet_osm_nodes -[hidden]> planet_osm_ways planet_osm_ways -[hidden]> planet_osm_rels planet_osm_ways -[hidden]-> place +place -[hidden]-> place_postcode +place -[hidden]-> place_interpolation planet_osm_nodes::id <- planet_osm_ways::nodes +planet_osm_nodes::id <- place_interpolation::nodes @enduml diff --git a/docs/develop/osm2pgsql-tables.svg b/docs/develop/osm2pgsql-tables.svg index 959380d1..15a77aff 100644 --- a/docs/develop/osm2pgsql-tables.svg +++ b/docs/develop/osm2pgsql-tables.svg @@ -1,8 +1,11 @@ -planet_osm_nodesidBIGINTlatINTlonINTplanet_osm_waysidBIGINTnodesBIGINT[]tagsTEXT[]planet_osm_relsidBIGINTpartsBIGINT[]membersTEXT[]tagsTEXT[]way_offSMALLINTrel_offSMALLINTplaceosm_typeCHAR(1)osm_idBIGINTclassTEXTtypeTEXTnameHSTOREaddressHSTOREextratagsHSTOREadmin_levelSMALLINTgeometryGEOMETRY