add BDD tests for DB

test/bdd/utils/checks.py

@@ -12,9 +12,10 @@ import re
 import math
 
 from psycopg import sql as pysql
-from psycopg.rows import dict_row, tuple_row
+from psycopg.rows import dict_row
 from .geometry_alias import ALIASES
 
+
 COMPARATOR_TERMS = {
     'exactly': lambda exp, act: exp == act,
     'more than': lambda exp, act: act > exp,
@@ -26,11 +27,19 @@ def _pretty(obj):
     return json.dumps(obj, sort_keys=True, indent=2)
 
 
+def _pt_close(p1, p2):
+    return math.isclose(p1[0], p2[0], abs_tol=1e-07) \
+           and math.isclose(p1[1], p2[1], abs_tol=1e-07)
+
+
 def within_box(value, expect):
     coord = [float(x) for x in expect.split(',')]
 
     if isinstance(value, str):
-        value = value.split(',')
+        if value.startswith('POINT'):
+            value = value[6:-1].split(' ')
+        else:
+            value = value.split(',')
     value = list(map(float, value))
 
     if len(value) == 2:
@@ -98,10 +107,10 @@ class ResultAttr:
                 self.subobj = self.subobj[sub]
 
     def __eq__(self, other):
-        if not isinstance(other, str):
-            raise NotImplementedError()
-
         # work around bad quoting by pytest-bdd
+        if not isinstance(other, str):
+            return self.subobj == other
+
         other = other.replace(r'\\', '\\')
 
         if self.fmt in COMPARISON_FUNCS:
@@ -148,18 +157,16 @@ class ResultAttr:
                      for pt in map(str.strip, m[2].split(','))]
 
         if expected.startswith('country:'):
-            ccode = geom[8:].upper()
+            ccode = expected[8:].upper()
             assert ccode in ALIASES, f"Geometry error: unknown country {ccode}"
-            return m[1] == 'POINT' and \
-                all(math.isclose(p1, p2) for p1, p2 in zip(converted[0], ALIASES[ccode]))
+            return m[1] == 'POINT' and _pt_close(converted[0], ALIASES[ccode])
 
         if ',' not in expected:
-            return m[1] == 'POINT' and \
-                all(math.isclose(p1, p2) for p1, p2 in zip(converted[0], self.get_point(expected)))
+            return m[1] == 'POINT' and _pt_close(converted[0], self.get_point(expected))
 
         if '(' not in expected:
             return m[1] == 'LINESTRING' and \
-                all(math.isclose(p1[0], p2[0]) and math.isclose(p1[1], p2[1]) for p1, p2 in
+                all(_pt_close(p1, p2) for p1, p2 in
                     zip(converted, (self.get_point(p) for p in expected.split(','))))
 
         if m[1] != 'POLYGON':
@@ -174,7 +181,7 @@ class ResultAttr:
                                "First and last point need to be the same")
         for line in (exp_coords[:-1], exp_coords[-1:0:-1]):
             for i in range(len(line)):
-                if all(math.isclose(p1[0], p2[0]) and math.isclose(p1[1], p2[1]) for p1, p2 in
+                if all(_pt_close(p1, p2) for p1, p2 in
                        zip(converted, line[i:] + line[:i])):
                     return True
 
@@ -199,7 +206,7 @@ def check_table_content(conn, tablename, data, grid=None, exact=False):
             cols.extend(('osm_id', 'osm_type'))
         elif '!' in col:
             name, fmt = col.rsplit('!', 1)
-            if fmt == 'wkt':
+            if fmt in ('wkt', 'in_box'):
                 cols.append(f"ST_AsText({name}) as {name}")
             else:
                 cols.append(name.split('+')[0])
@@ -215,7 +222,7 @@ def check_table_content(conn, tablename, data, grid=None, exact=False):
             table_content += '\n' + str(row)
             for i in lines:
                 for col, value in zip(data[0], data[i]):
-                    if ResultAttr(row, col, grid=grid) != value:
+                    if ResultAttr(row, col, grid=grid) != (None if value == '-' else value):
                         break
                 else:
                     lines.remove(i)
@@ -228,15 +235,3 @@ def check_table_content(conn, tablename, data, grid=None, exact=False):
                + '\n'.join(str(data[i]) for i in lines) \
                + "\nTable content:\n" \
                + table_content
-
-
-def check_table_has_lines(conn, tablename, osm_type, osm_id, osm_class):
-    sql = pysql.SQL("""SELECT count(*) FROM {}
-                       WHERE osm_type = %s and osm_id = %s""").format(pysql.Identifier(tablename))
-    params = [osm_type, int(osm_id)]
-    if osm_class:
-        sql += pysql.SQL(' AND class = %s')
-        params.append(osm_class)
-
-    with conn.cursor(row_factory=tuple_row) as cur:
-        assert cur.execute(sql, params).fetchone()[0] == 0

test/bdd/utils/db.py

@@ -13,7 +13,7 @@ from psycopg import sql as pysql
 
 from nominatim_db.tools.database_import import setup_database_skeleton, create_tables, \
                                                create_partition_tables, create_search_indices
-from nominatim_db.data.country_info import setup_country_tables
+from nominatim_db.data.country_info import setup_country_tables, create_country_names
 from nominatim_db.tools.refresh import create_functions, load_address_levels_from_config
 from nominatim_db.tools.exec_utils import run_osm2pgsql
 from nominatim_db.tokenizer import factory as tokenizer_factory
@@ -98,4 +98,5 @@ class DBManager:
             create_functions(conn, config, enable_diff_updates=False)
             asyncio.run(create_search_indices(conn, config))
 
-        tokenizer_factory.create_tokenizer(config)
+            tokenizer = tokenizer_factory.create_tokenizer(config)
+            create_country_names(conn, tokenizer)

test/bdd/utils/grid.py

@@ -38,7 +38,7 @@ class Grid:
         """
         value = value.strip()
         if ' ' in value:
-            return [int(v) for v in value.split(' ', 1)]
+            return [float(v) for v in value.split(' ', 1)]
 
         return self.grid.get(value)
 

test/bdd/utils/place_inserter.py

@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2025 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Helper classes for filling the place table.
+"""
+import random
+import string
+
+from .geometry_alias import ALIASES
+
+
+class PlaceColumn:
+    """ Helper class to collect contents from a BDD table row and
+        insert it into the place table.
+    """
+    def __init__(self, grid=None):
+        self.columns = {'admin_level': 15}
+        self.grid = grid
+        self.geometry = None
+
+    def add_row(self, headings, row, force_name):
+        """ Parse the content from the given behave row as place column data.
+        """
+        for name, value in zip(headings, row):
+            self._add(name, value)
+
+        assert 'osm_type' in self.columns, "osm column missing"
+
+        if force_name and 'name' not in self.columns:
+            self._add_hstore(
+                'name',
+                'name',
+                ''.join(random.choices(string.printable, k=random.randrange(30))),
+            )
+
+        return self
+
+    def _add(self, key, value):
+        if hasattr(self, '_set_key_' + key):
+            getattr(self, '_set_key_' + key)(value)
+        elif key.startswith('name+'):
+            self._add_hstore('name', key[5:], value)
+        elif key.startswith('extra+'):
+            self._add_hstore('extratags', key[6:], value)
+        elif key.startswith('addr+'):
+            self._add_hstore('address', key[5:], value)
+        elif key in ('name', 'address', 'extratags'):
+            self.columns[key] = eval('{' + value + '}')
+        else:
+            assert key in ('class', 'type'), "Unknown column '{}'.".format(key)
+            self.columns[key] = None if value == '' else value
+
+    def _set_key_name(self, value):
+        self._add_hstore('name', 'name', value)
+
+    def _set_key_osm(self, value):
+        assert value[0] in 'NRW' and value[1:].isdigit(), \
+               "OSM id needs to be of format <NRW><id>."
+
+        self.columns['osm_type'] = value[0]
+        self.columns['osm_id'] = int(value[1:])
+
+    def _set_key_admin(self, value):
+        self.columns['admin_level'] = int(value)
+
+    def _set_key_housenr(self, value):
+        if value:
+            self._add_hstore('address', 'housenumber', value)
+
+    def _set_key_postcode(self, value):
+        if value:
+            self._add_hstore('address', 'postcode', value)
+
+    def _set_key_street(self, value):
+        if value:
+            self._add_hstore('address', 'street', value)
+
+    def _set_key_addr_place(self, value):
+        if value:
+            self._add_hstore('address', 'place', value)
+
+    def _set_key_country(self, value):
+        if value:
+            self._add_hstore('address', 'country', value)
+
+    def _set_key_geometry(self, value):
+        if value.startswith('country:'):
+            ccode = value[8:].upper()
+            self.geometry = "ST_SetSRID(ST_Point({}, {}), 4326)".format(*ALIASES[ccode])
+        elif ',' not in value:
+            if self.grid:
+                pt = self.grid.parse_point(value)
+            else:
+                pt = value.split(' ')
+            self.geometry = f"ST_SetSRID(ST_Point({pt[0]}, {pt[1]}), 4326)"
+        elif '(' not in value:
+            if self.grid:
+                coords = ','.join(' '.join(f"{p:.7f}" for p in pt)
+                                  for pt in self.grid.parse_line(value))
+            else:
+                coords = value
+            self.geometry = f"'srid=4326;LINESTRING({coords})'::geometry"
+        else:
+            if self.grid:
+                coords = ','.join(' '.join(f"{p:.7f}" for p in pt)
+                                  for pt in self.grid.parse_line(value[1:-1]))
+            else:
+                coords = value[1:-1]
+            self.geometry = f"'srid=4326;POLYGON(({coords}))'::geometry"
+
+    def _add_hstore(self, column, key, value):
+        if column in self.columns:
+            self.columns[column][key] = value
+        else:
+            self.columns[column] = {key: value}
+
+    def db_delete(self, cursor):
+        """ Issue a delete for the given OSM object.
+        """
+        cursor.execute('DELETE FROM place WHERE osm_type = %s and osm_id = %s',
+                       (self.columns['osm_type'], self.columns['osm_id']))
+
+    def db_insert(self, cursor):
+        """ Insert the collected data into the database.
+        """
+        if self.columns['osm_type'] == 'N' and self.geometry is None:
+            pt = self.grid.get(str(self.columns['osm_id'])) if self.grid else None
+            if pt is None:
+                pt = (random.uniform(-180, 180), random.uniform(-90, 90))
+
+            self.geometry = "ST_SetSRID(ST_Point({}, {}), 4326)".format(*pt)
+        else:
+            assert self.geometry is not None, "Geometry missing"
+
+        query = 'INSERT INTO place ({}, geometry) values({}, {})'.format(
+            ','.join(self.columns.keys()),
+            ','.join(['%s' for x in range(len(self.columns))]),
+            self.geometry)
+        cursor.execute(query, list(self.columns.values()))