From 3bcd1aa72178e79a8fa3e4cbfbff7c310a235d34 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 23 Dec 2025 22:26:43 +0100 Subject: [PATCH] adapt BDD tests for new postcode table structure --- .../bdd/features/db/import/addressing.feature | 27 ---- test/bdd/features/db/import/linking.feature | 15 -- test/bdd/features/db/import/placex.feature | 17 -- test/bdd/features/db/import/postcodes.feature | 35 ++-- .../features/db/query/normalization.feature | 13 -- test/bdd/features/db/query/postcodes.feature | 21 +-- test/bdd/features/db/update/naming.feature | 20 --- test/bdd/features/db/update/postcode.feature | 153 ++++++++++-------- test/bdd/test_db.py | 44 ++++- test/bdd/utils/grid.py | 26 +++ test/bdd/utils/place_inserter.py | 26 +-- 11 files changed, 167 insertions(+), 230 deletions(-) delete mode 100644 test/bdd/features/db/update/naming.feature diff --git a/test/bdd/features/db/import/addressing.feature b/test/bdd/features/db/import/addressing.feature index e61a4777..82ca1e6f 100644 --- a/test/bdd/features/db/import/addressing.feature +++ b/test/bdd/features/db/import/addressing.feature @@ -268,33 +268,6 @@ Feature: Address computation | W93 | R34 | | W93 | R4 | - Scenario: postcode boundaries do appear in the address of a way - Given the grid with origin DE - | 1 | | | | | 8 | | 6 | | 2 | - | |10 |11 | | | | | | | | - | |13 |12 | | | | | | | | - | 20| | | 21| | | | | | | - | | | | | | | | | | | - | | | | | | 9 | | | | | - | 4 | | | | | | | 7 | | 3 | - And the named places - | osm | class | type | admin | addr+postcode | geometry | - | R1 | boundary | administrative | 6 | 10000 | (1,2,3,4,1) | - | R34 | boundary | administrative | 8 | 11000 | (1,6,7,4,1) | - And the places - | osm | class | type | addr+postcode | geometry | - | R4 | boundary | postal_code | 11200 | (1,8,9,4,1) | - And the named places - | osm | class | type | geometry | - | W93 | highway | residential | 20,21 | - And the places - | osm | class | type | addr+postcode | geometry | - | W22 | place | postcode | 11234 | (10,11,12,13,10) | - When importing - Then place_addressline contains - | object | address | - | W93 | R4 | - Scenario: squares do not appear in the address of a street Given the grid | | 1 | | 2 | | diff --git a/test/bdd/features/db/import/linking.feature b/test/bdd/features/db/import/linking.feature index 22d5d48e..910a09d1 100644 --- a/test/bdd/features/db/import/linking.feature +++ b/test/bdd/features/db/import/linking.feature @@ -16,21 +16,6 @@ Feature: Linking of places | R13 | - | | N256 | - | - Scenario: Postcode areas cannot be linked - Given the grid with origin US - | 1 | | 2 | - | | 9 | | - | 4 | | 3 | - And the named places - | osm | class | type | addr+postcode | extra+wikidata | geometry | - | R13 | boundary | postal_code | 12345 | Q87493 | (1,2,3,4,1) | - | N25 | place | suburb | 12345 | Q87493 | 9 | - When importing - Then placex contains - | object | linked_place_id | - | R13 | - | - | N25 | - | - Scenario: Waterways are linked when in waterway relations Given the grid | 1 | | | | 3 | 4 | | | | 6 | diff --git a/test/bdd/features/db/import/placex.feature b/test/bdd/features/db/import/placex.feature index 8b4a07db..dfda880d 100644 --- a/test/bdd/features/db/import/placex.feature +++ b/test/bdd/features/db/import/placex.feature @@ -46,23 +46,6 @@ Feature: Import into placex | object | admin_level | | N1 | 3 | - Scenario: postcode node without postcode is dropped - Given the places - | osm | class | type | name+ref | - | N1 | place | postcode | 12334 | - When importing - Then placex has no entry for N1 - - Scenario: postcode boundary without postcode is dropped - Given the 0.01 grid - | 1 | 2 | - | 3 | | - Given the places - | osm | class | type | name+ref | geometry | - | R1 | boundary | postal_code | 554476 | (1,2,3,1) | - When importing - Then placex has no entry for R1 - Scenario: search and address ranks for boundaries are correctly assigned Given the named places | osm | class | type | diff --git a/test/bdd/features/db/import/postcodes.feature b/test/bdd/features/db/import/postcodes.feature index 7f69b1e1..f3c3b122 100644 --- a/test/bdd/features/db/import/postcodes.feature +++ b/test/bdd/features/db/import/postcodes.feature @@ -121,8 +121,8 @@ Feature: Import of postcodes | | 1 | 2 | | | | | 4 | 3 | | | And the named places - | osm | class | type | geometry | - | W93 | highway | pedestriant | (10,11,12,13,10) | + | osm | class | type | geometry | + | W93 | highway | pedestrian | (10,11,12,13,10) | And the named places | osm | class | type | addr+postcode | geometry | | W22 | building | yes | 45023 | (1,2,3,4,1) | @@ -134,14 +134,13 @@ Feature: Import of postcodes Scenario: Roads get postcodes from nearby unnamed buildings without other info Given the grid with origin US | 10 | | | | 11 | - | | 1 | 2 | | | - | | 4 | 3 | | | + | | 1 | | | | And the named places | osm | class | type | geometry | | W93 | highway | residential | 10,11 | - And the places - | osm | class | type | addr+postcode | geometry | - | W22 | place | postcode | 45023 | (1,2,3,4,1) | + And the postcodes + | osm | postcode | centroid | + | W22 | 45023 | 1 | When importing Then placex contains | object | postcode | @@ -172,26 +171,12 @@ Feature: Import of postcodes Scenario: Postcodes are added to the postcode Given the places | osm | class | type | addr+postcode | addr+housenumber | geometry | - | N34 | place | house | 01982 | 111 |country:de | + | N34 | place | house | 01982 | 111 | country:de | When importing - Then location_postcode contains exactly - | country_code | postcode | geometry!wkt | + Then location_postcodes contains exactly + | country_code | postcode | centroid!wkt | | de | 01982 | country:de | - @skip - Scenario: search and address ranks for GB post codes correctly assigned - Given the places - | osm | class | type | postcode | geometry | - | N1 | place | postcode | E45 2CD | country:gb | - | N2 | place | postcode | E45 2 | country:gb | - | N3 | place | postcode | Y45 | country:gb | - When importing - Then location_postcode contains exactly - | postcode | country_code | rank_search | rank_address | - | E45 2CD | gb | 25 | 5 | - | E45 2 | gb | 23 | 5 | - | Y45 | gb | 21 | 5 | - Scenario: Postcodes outside all countries are not added to the postcode table Given the places | osm | class | type | addr+postcode | addr+housenumber | addr+place | geometry | @@ -200,7 +185,7 @@ Feature: Import of postcodes | osm | class | type | name | geometry | | N1 | place | hamlet | Null Island | 0 0 | When importing - Then location_postcode contains exactly + Then location_postcodes contains exactly | place_id | When geocoding "111, 01982 Null Island" Then the result set contains diff --git a/test/bdd/features/db/query/normalization.feature b/test/bdd/features/db/query/normalization.feature index f884be6b..aad5b3db 100644 --- a/test/bdd/features/db/query/normalization.feature +++ b/test/bdd/features/db/query/normalization.feature @@ -154,19 +154,6 @@ Feature: Import and search of names | object | | R2 | - Scenario: Postcode boundaries without ref - Given the grid with origin FR - | | 2 | | - | 1 | | 3 | - Given the places - | osm | class | type | postcode | geometry | - | R1 | boundary | postal_code | 123-45 | (1,2,3,1) | - When importing - When geocoding "123-45" - Then result 0 contains - | object | - | R1 | - Scenario Outline: Housenumbers with special characters are found Given the grid | 1 | | | | 2 | diff --git a/test/bdd/features/db/query/postcodes.feature b/test/bdd/features/db/query/postcodes.feature index 819ea061..3880f664 100644 --- a/test/bdd/features/db/query/postcodes.feature +++ b/test/bdd/features/db/query/postcodes.feature @@ -78,8 +78,8 @@ Feature: Querying fo postcode variants | N34 | place | house | EH4 7EA | 111 | country:gb | | N35 | place | house | E4 7EA | 111 | country:gb | When importing - Then location_postcode contains exactly - | country_code | postcode | geometry!wkt | + Then location_postcodes contains exactly + | country_code | postcode | centroid!wkt | | gb | EH4 7EA | country:gb | | gb | E4 7EA | country:gb | When geocoding "EH4 7EA" @@ -90,20 +90,3 @@ Feature: Querying fo postcode variants Then result 0 contains | type | display_name | | postcode | E4 7EA, United Kingdom | - - - Scenario: Postcode areas are preferred over postcode points - Given the grid with origin DE - | 1 | 2 | - | 4 | 3 | - Given the places - | osm | class | type | postcode | geometry | - | R23 | boundary | postal_code | 12345 | (1,2,3,4,1) | - When importing - Then location_postcode contains exactly - | country_code | postcode | - | de | 12345 | - When geocoding "12345, de" - Then result 0 contains - | object | - | R23 | diff --git a/test/bdd/features/db/update/naming.feature b/test/bdd/features/db/update/naming.feature deleted file mode 100644 index 2912a7da..00000000 --- a/test/bdd/features/db/update/naming.feature +++ /dev/null @@ -1,20 +0,0 @@ -Feature: Update of names in place objects - Test all naming related issues in updates - - Scenario: Delete postcode from postcode boundaries without ref - Given the grid with origin DE - | 1 | 2 | - | 4 | 3 | - Given the places - | osm | class | type | postcode | geometry | - | R1 | boundary | postal_code | 123-45 | (1,2,3,4,1) | - When importing - And geocoding "123-45" - Then result 0 contains - | object | - | R1 | - When updating places - | osm | class | type | geometry | - | R1 | boundary | postal_code | (1,2,3,4,1) | - Then placex has no entry for R1 - diff --git a/test/bdd/features/db/update/postcode.feature b/test/bdd/features/db/update/postcode.feature index d62953e7..c2a1b7f8 100644 --- a/test/bdd/features/db/update/postcode.feature +++ b/test/bdd/features/db/update/postcode.feature @@ -2,20 +2,22 @@ Feature: Update of postcode Tests for updating of data related to postcodes Scenario: Updating postcode in postcode boundaries without ref - Given the grid - | 1 | 2 | - | 4 | 3 | - Given the places - | osm | class | type | postcode | geometry | - | R1 | boundary | postal_code | 12345 | (1,2,3,4,1) | + Given the grid with origin FR + | 1 | | 2 | + | | 9 | | + | 4 | | 3 | + Given the postcodes + | osm | postcode | centroid | geometry | + | R1 | 12345 | 9 | (1,2,3,4,1) | When importing And geocoding "12345" Then result 0 contains | object | | R1 | - When updating places - | osm | class | type | postcode | geometry | - | R1 | boundary | postal_code | 54321 | (1,2,3,4,1) | + Given the postcodes + | osm | postcode | centroid | geometry | + | R1 | 54321 | 9 | (1,2,3,4,1) | + When refreshing postcodes And geocoding "12345" Then exactly 0 results are returned When geocoding "54321" @@ -28,17 +30,21 @@ Feature: Update of postcode | osm | class | type | addr+postcode | addr+housenumber | geometry | | N34 | place | house | 01982 | 111 | country:de | When importing - Then location_postcode contains exactly - | country_code | postcode | geometry!wkt | + Then location_postcodes contains exactly + | country_code | postcode | centroid!wkt | | de | 01982 | country:de | + Given the postcodes + | osm | postcode | centroid | + | N66 | 99201 | country:fr | When updating places | osm | class | type | addr+postcode | addr+housenumber | geometry | | N35 | place | house | 4567 | 5 | country:ch | - And updating postcodes - Then location_postcode contains exactly - | country_code | postcode | geometry!wkt | + And refreshing postcodes + Then location_postcodes contains exactly + | country_code | postcode | centroid!wkt | | de | 01982 | country:de | | ch | 4567 | country:ch | + | fr | 99201 | country:fr | Scenario: When the last postcode is deleted, it is deleted from postcode Given the places @@ -47,9 +53,9 @@ Feature: Update of postcode | N35 | place | house | 4567 | 5 | country:ch | When importing And marking for delete N34 - And updating postcodes - Then location_postcode contains exactly - | country_code | postcode | geometry!wkt | + And refreshing postcodes + Then location_postcodes contains exactly + | country_code | postcode | centroid!wkt | | ch | 4567 | country:ch | Scenario: A postcode is not deleted from postcode when it exist in another country @@ -59,64 +65,24 @@ Feature: Update of postcode | N35 | place | house | 01982 | 5 | country:fr | When importing And marking for delete N34 - And updating postcodes - Then location_postcode contains exactly - | country_code | postcode | geometry!wkt| + And refreshing postcodes + Then location_postcodes contains exactly + | country_code | postcode | centroid!wkt| | fr | 01982 | country:fr | Scenario: Updating a postcode is reflected in postcode table Given the places - | osm | class | type | addr+postcode | geometry | + | osm | class | type | addr+postcode | geometry | | N34 | place | postcode | 01982 | country:de | When importing And updating places | osm | class | type | addr+postcode | geometry | | N34 | place | postcode | 20453 | country:de | - And updating postcodes - Then location_postcode contains exactly - | country_code | postcode | geometry!wkt | + And refreshing postcodes + Then location_postcodes contains exactly + | country_code | postcode | centroid!wkt | | de | 20453 | country:de | - Scenario: When changing from a postcode type, the entry appears in placex - When importing - And updating places - | osm | class | type | addr+postcode | geometry | - | N34 | place | postcode | 01982 | country:de | - Then placex has no entry for N34 - When updating places - | osm | class | type | addr+postcode | housenr | geometry | - | N34 | place | house | 20453 | 1 | country:de | - Then placex contains - | object | addr+housenumber | geometry!wkt | - | N34 | 1 | country:de | - And place contains exactly - | osm_type | osm_id | class | type | - | N | 34 | place | house | - When updating postcodes - Then location_postcode contains exactly - | country_code | postcode | geometry!wkt | - | de | 20453 | country:de | - - Scenario: When changing to a postcode type, the entry disappears from placex - When importing - And updating places - | osm | class | type | addr+postcode | housenr | geometry | - | N34 | place | house | 20453 | 1 | country:de | - Then placex contains - | object | addr+housenumber | geometry!wkt | - | N34 | 1 | country:de| - When updating places - | osm | class | type | addr+postcode | geometry | - | N34 | place | postcode | 01982 | country:de | - Then placex has no entry for N34 - And place contains exactly - | osm_type | osm_id | class | type | - | N | 34 | place | postcode | - When updating postcodes - Then location_postcode contains exactly - | country_code | postcode | geometry!wkt | - | de | 01982 | country:de | - Scenario: When a parent is deleted, the postcode gets a new parent Given the grid with origin DE | 1 | | 3 | 4 | @@ -126,14 +92,59 @@ Feature: Update of postcode | osm | class | type | name | admin | geometry | | R1 | boundary | administrative | Big | 6 | (1,4,6,2,1) | | R2 | boundary | administrative | Small | 6 | (1,3,5,2,1) | - Given the places - | osm | class | type | addr+postcode | geometry | - | N9 | place | postcode | 12345 | 9 | + Given the postcodes + | osm | postcode | centroid | + | N9 | 12345 | 9 | When importing - Then location_postcode contains exactly - | postcode | geometry!wkt | parent_place_id | + Then location_postcodes contains exactly + | postcode | centroid!wkt | parent_place_id | | 12345 | 9 | R2 | When marking for delete R2 - Then location_postcode contains exactly - | country_code | postcode | geometry!wkt | parent_place_id | + Then location_postcodes contains exactly + | country_code | postcode | centroid!wkt | parent_place_id | | de | 12345 | 9 | R1 | + + Scenario: When a postcode area appears, postcode points are shadowed + Given the grid with origin DE + | 1 | | 3 | | + | | 9 | | 8 | + | 2 | | 5 | | + Given the postcodes + | osm | postcode | centroid | + | N92 | 44321 | 9 | + | N4 | 00245 | 8 | + When importing + Then location_postcodes contains exactly + | country_code | postcode | osm_id | centroid!wkt | + | de | 44321 | - | 9 | + | de | 00245 | - | 8 | + Given the postcodes + | osm | postcode | centroid | geometry | + | R45 | 00245 | 9 | (1,3,5,2,1) | + When refreshing postcodes + Then location_postcodes contains exactly + | country_code | postcode | osm_id | centroid!wkt | + | de | 00245 | 45 | 9 | + + Scenario: When a postcode area disappears, postcode points are unshadowed + Given the grid with origin DE + | 1 | | 3 | | + | | 9 | | 8 | + | 2 | | 5 | | + Given the postcodes + | osm | postcode | centroid | geometry | + | R45 | 00245 | 9 | (1,3,5,2,1) | + Given the postcodes + | osm | postcode | centroid | + | N92 | 44321 | 9 | + | N4 | 00245 | 8 | + When importing + Then location_postcodes contains exactly + | country_code | postcode | osm_id | centroid!wkt | + | de | 00245 | 45 | 9 | + When marking for delete R45 + And refreshing postcodes + Then location_postcodes contains exactly + | country_code | postcode | osm_id | centroid!wkt | + | de | 44321 | - | 9 | + | de | 00245 | - | 8 | diff --git a/test/bdd/test_db.py b/test/bdd/test_db.py index 30e1e2bb..1a7eef7c 100644 --- a/test/bdd/test_db.py +++ b/test/bdd/test_db.py @@ -11,6 +11,7 @@ These tests check the Nominatim import chain after the osm2pgsql import. """ import asyncio import re +from collections import defaultdict import psycopg @@ -20,6 +21,7 @@ from pytest_bdd.parsers import re as step_parse from utils.place_inserter import PlaceColumn from utils.checks import check_table_content +from utils.geometry_alias import ALIASES from nominatim_db.config import Configuration from nominatim_db import cli @@ -97,6 +99,41 @@ def import_place_entrances(db_conn, datatable, node_grid): data.columns.get('extratags'))) +@given(step_parse('the postcodes'), target_fixture=None) +def import_place_postcode(db_conn, datatable, node_grid): + """ Insert todo rows into the place_postcode table. If a row for the + requested object already exists it is overwritten. + """ + with db_conn.cursor() as cur: + for row in datatable[1:]: + data = defaultdict(lambda: None) + data.update((k, v) for k, v in zip(datatable[0], row)) + + if data['centroid'].startswith('country:'): + ccode = data['centroid'][8:].upper() + data['centroid'] = 'srid=4326;POINT({} {})'.format(*ALIASES[ccode]) + else: + data['centroid'] = f"srid=4326;{node_grid.geometry_to_wkt(data['centroid'])}" + + data['osm_type'] = data['osm'][0] + data['osm_id'] = data['osm'][1:] + + if 'geometry' in data: + geom = f"'srid=4326;{node_grid.geometry_to_wkt(data['geometry'])}'::geometry" + else: + geom = 'null' + + cur.execute(""" DELETE FROM place_postcode + WHERE osm_type = %(osm_type)s and osm_id = %(osm_id)s""", + data) + cur.execute(f"""INSERT INTO place_postcode + (osm_type, osm_id, country_code, postcode, centroid, geometry) + VALUES (%(osm_type)s, %(osm_id)s, + %(country)s, %(postcode)s, + %(centroid)s, {geom})""", data) + db_conn.commit() + + @given('the ways', target_fixture=None) def import_ways(db_conn, datatable): """ Import raw ways into the osm2pgsql way middle table. @@ -168,7 +205,7 @@ def do_update(db_conn, update_config, node_grid, datatable): @when('updating entrances', target_fixture=None) def update_place_entrances(db_conn, datatable, node_grid): - """ Insert todo rows into the place_entrance table. + """ Update rows in the place_entrance table. """ with db_conn.cursor() as cur: for row in datatable[1:]: @@ -181,9 +218,10 @@ def update_place_entrances(db_conn, datatable, node_grid): VALUES (%s, %s, %s, {})""".format(data.get_wkt()), (data.columns['osm_id'], data.columns['type'], data.columns.get('extratags'))) + db_conn.commit() -@when('updating postcodes') +@when('refreshing postcodes') def do_postcode_update(update_config): """ Recompute the postcode centroids. """ @@ -203,6 +241,8 @@ def do_delete_place(db_conn, update_config, node_grid, otype, oid): if otype == 'N': cur.execute('DELETE FROM place_entrance WHERE osm_id = %s', (oid, )) + cur.execute('DELETE FROM place_postcode WHERE osm_type = %s and osm_id = %s', + (otype, oid)) db_conn.commit() cli.nominatim(['index', '-q'], update_config.environ) diff --git a/test/bdd/utils/grid.py b/test/bdd/utils/grid.py index 50355a1b..a97fa767 100644 --- a/test/bdd/utils/grid.py +++ b/test/bdd/utils/grid.py @@ -8,6 +8,7 @@ A grid describing node placement in an area. Useful for visually describing geometries. """ +import re class Grid: @@ -44,3 +45,28 @@ class Grid: def parse_line(self, value): return [self.parse_point(p) for p in value.split(',')] + + def geometry_to_wkt(self, value): + """ Parses the given value into a geometry and returns the WKT. + + The value can either be a WKT already or a geometry shortcut + with coordinates or grid points. + """ + if re.fullmatch(r'([A-Z]+)\((.*)\)', value) is not None: + return value # already a WKT + + # points + if ',' not in value: + x, y = self.parse_point(value) + return f"POINT({x} {y})" + + # linestring + if '(' not in value: + coords = ','.join(' '.join(f"{p:.7f}" for p in pt) + for pt in self.parse_line(value)) + return f"LINESTRING({coords})" + + # simple polygons + coords = ','.join(' '.join(f"{p:.7f}" for p in pt) + for pt in self.parse_line(value[1:-1])) + return f"POLYGON(({coords}))" diff --git a/test/bdd/utils/place_inserter.py b/test/bdd/utils/place_inserter.py index a3ddf5c7..086693f4 100644 --- a/test/bdd/utils/place_inserter.py +++ b/test/bdd/utils/place_inserter.py @@ -11,6 +11,7 @@ import random import string from .geometry_alias import ALIASES +from .grid import Grid class PlaceColumn: @@ -19,7 +20,7 @@ class PlaceColumn: """ def __init__(self, grid=None): self.columns = {'admin_level': 15} - self.grid = grid + self.grid = grid or Grid() self.geometry = None def add_row(self, headings, row, force_name): @@ -91,26 +92,9 @@ class PlaceColumn: if value.startswith('country:'): ccode = value[8:].upper() self.geometry = "ST_SetSRID(ST_Point({}, {}), 4326)".format(*ALIASES[ccode]) - elif ',' not in value: - if self.grid: - pt = self.grid.parse_point(value) - else: - pt = value.split(' ') - self.geometry = f"ST_SetSRID(ST_Point({pt[0]}, {pt[1]}), 4326)" - elif '(' not in value: - if self.grid: - coords = ','.join(' '.join(f"{p:.7f}" for p in pt) - for pt in self.grid.parse_line(value)) - else: - coords = value - self.geometry = f"'srid=4326;LINESTRING({coords})'::geometry" else: - if self.grid: - coords = ','.join(' '.join(f"{p:.7f}" for p in pt) - for pt in self.grid.parse_line(value[1:-1])) - else: - coords = value[1:-1] - self.geometry = f"'srid=4326;POLYGON(({coords}))'::geometry" + wkt = self.grid.geometry_to_wkt(value) + self.geometry = f"'srid=4326;{wkt}'::geometry" def _add_hstore(self, column, key, value): if column in self.columns: @@ -120,7 +104,7 @@ class PlaceColumn: def get_wkt(self): if self.columns['osm_type'] == 'N' and self.geometry is None: - pt = self.grid.get(str(self.columns['osm_id'])) if self.grid else None + pt = self.grid.get(str(self.columns['osm_id'])) if pt is None: pt = (random.uniform(-180, 180), random.uniform(-90, 90))