hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-14 23:14:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

enable flake for bdd test code

Browse Source
This commit is contained in:
Sarah Hoffmann
2025-03-09 17:34:04 +01:00
parent c70dfccaca
commit 78f839fbd3
15 changed files with 396 additions and 393 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.flake8
View File

@@ -8,3 +8,4 @@ per-file-ignores =
__init__.py: F401
test/python/utils/test_json_writer.py: E131
test/python/conftest.py: E402
test/bdd/*: F821

2
.github/workflows/ci-tests.yml vendored
View File

@@ -100,7 +100,7 @@ jobs:
run: ./venv/bin/pip install -U flake8
- name: Python linting
run: ../venv/bin/python -m flake8 src test/python
run: ../venv/bin/python -m flake8 src test/python test/bdd
working-directory: Nominatim
- name: Install mypy and typechecking info

2
Makefile
View File

@@ -24,7 +24,7 @@ pytest:
pytest test/python
lint:
flake8 src test/python
flake8 src test/python test/bdd
bdd:
cd test/bdd; behave -DREMOVE_TEMPLATE=1

15
test/bdd/environment.py
View File

@@ -2,17 +2,17 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2024 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
from pathlib import Path
import sys
from behave import *
from behave import * # noqa
sys.path.insert(1, str(Path(__file__, '..', '..', '..', 'src').resolve()))
from steps.geometry_factory import GeometryFactory
from steps.nominatim_environment import NominatimEnvironment
from steps.geometry_factory import GeometryFactory # noqa: E402
from steps.nominatim_environment import NominatimEnvironment # noqa: E402
TEST_BASE_DIR = Path(__file__, '..', '..').resolve()
@@ -32,7 +32,9 @@ userconfig = {
'API_ENGINE': 'falcon'
}
use_step_matcher("re")
use_step_matcher("re") # noqa: F405
def before_all(context):
# logging setup
@@ -46,7 +48,7 @@ def before_all(context):
def before_scenario(context, scenario):
if not 'SQLITE' in context.tags \
if 'SQLITE' not in context.tags \
and context.config.userdata['API_TEST_DB'].startswith('sqlite:'):
context.scenario.skip("Not usable with Sqlite database.")
elif 'DB' in context.tags:
@@ -56,6 +58,7 @@ def before_scenario(context, scenario):
elif 'UNKNOWNDB' in context.tags:
context.nominatim.setup_unknown_db()
def after_scenario(context, scenario):
if 'DB' in context.tags:
context.nominatim.teardown_db(context)

7
test/bdd/steps/check_functions.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2023 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Collection of assertion functions used for the steps.
@@ -11,6 +11,7 @@ import json
import math
import re
OSM_TYPE = {'N': 'node', 'W': 'way', 'R': 'relation',
'n': 'node', 'w': 'way', 'r': 'relation',
'node': 'n', 'way': 'w', 'relation': 'r'}
@@ -23,11 +24,9 @@ class OsmType:
def __init__(self, value):
self.value = value
def __eq__(self, other):
return other == self.value or other == OSM_TYPE[self.value]
def __str__(self):
return f"{self.value} or {OSM_TYPE[self.value]}"
@@ -81,7 +80,6 @@ class Bbox:
return str(self.coord)
def check_for_attributes(obj, attrs, presence='present'):
""" Check that the object has the given attributes. 'attrs' is a
string with a comma-separated list of attributes. If 'presence'
@@ -99,4 +97,3 @@ def check_for_attributes(obj, attrs, presence='present'):
else:
assert attr in obj, \
f"No attribute '{attr}'. Full response:\n{_dump_json()}"

2
test/bdd/steps/geometry_alias.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Collection of aliases for various world coordinates.

10
test/bdd/steps/geometry_factory.py
View File

@@ -2,13 +2,11 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
from pathlib import Path
import os
from steps.geometry_alias import ALIASES
class GeometryFactory:
""" Provides functions to create geometries from coordinates and data grids.
"""
@@ -47,7 +45,6 @@ class GeometryFactory:
return "ST_SetSRID('{}'::geometry, 4326)".format(out)
def mk_wkt_point(self, point):
""" Parse a point description.
The point may either consist of 'x y' coordinates or a number
@@ -65,7 +62,6 @@ class GeometryFactory:
assert pt is not None, "Scenario error: Point '{}' not found in grid".format(geom)
return "{} {}".format(*pt)
def mk_wkt_points(self, geom):
""" Parse a list of points.
The list must be a comma-separated list of points. Points
@@ -73,7 +69,6 @@ class GeometryFactory:
"""
return ','.join([self.mk_wkt_point(x) for x in geom.split(',')])
def set_grid(self, lines, grid_step, origin=(0.0, 0.0)):
""" Replace the grid with one from the given lines.
"""
@@ -87,7 +82,6 @@ class GeometryFactory:
x += grid_step
y += grid_step
def grid_node(self, nodeid):
""" Get the coordinates for the given grid node.
"""

12
test/bdd/steps/http_responses.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2023 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Classes wrapping HTTP responses from the Nominatim API.
@@ -45,7 +45,6 @@ class GenericResponse:
else:
self.result = [self.result]
def _parse_geojson(self):
self._parse_json()
if self.result:
@@ -76,7 +75,6 @@ class GenericResponse:
new['__' + k] = v
self.result.append(new)
def _parse_geocodejson(self):
self._parse_geojson()
if self.result:
@@ -87,7 +85,6 @@ class GenericResponse:
inner = r.pop('geocoding')
r.update(inner)
def assert_address_field(self, idx, field, value):
""" Check that result rows`idx` has a field `field` with value `value`
in its address. If idx is None, then all results are checked.
@@ -103,7 +100,6 @@ class GenericResponse:
address = self.result[idx]['address']
self.check_row_field(idx, field, value, base=address)
def match_row(self, row, context=None, field=None):
""" Match the result fields against the given behave table row.
"""
@@ -139,7 +135,6 @@ class GenericResponse:
else:
self.check_row_field(i, name, Field(value), base=subdict)
def check_row(self, idx, check, msg):
""" Assert for the condition 'check' and print 'msg' on fail together
with the contents of the failing result.
@@ -154,7 +149,6 @@ class GenericResponse:
assert check, _RowError(self.result[idx])
def check_row_field(self, idx, field, expected, base=None):
""" Check field 'field' of result 'idx' for the expected value
and print a meaningful error if the condition fails.
@@ -172,7 +166,6 @@ class GenericResponse:
f"\nBad value for field '{field}'. Expected: {expected}, got: {value}")
class SearchResponse(GenericResponse):
""" Specialised class for search and lookup responses.
Transforms the xml response in a format similar to json.
@@ -240,7 +233,8 @@ class ReverseResponse(GenericResponse):
assert 'namedetails' not in self.result[0], "More than one namedetails in result"
self.result[0]['namedetails'] = {}
for tag in child:
assert len(tag) == 0, f"Namedetails element '{tag.attrib['desc']}' has subelements"
assert len(tag) == 0, \
f"Namedetails element '{tag.attrib['desc']}' has subelements"
self.result[0]['namedetails'][tag.attrib['desc']] = tag.text
elif child.tag == 'geokml':
assert 'geokml' not in self.result[0], "More than one geokml in result"

25
test/bdd/steps/nominatim_environment.py
View File

@@ -2,10 +2,9 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2024 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
from pathlib import Path
import importlib
import tempfile
import psycopg
@@ -13,10 +12,9 @@ from psycopg import sql as pysql
from nominatim_db import cli
from nominatim_db.config import Configuration
from nominatim_db.db.connection import Connection, register_hstore, execute_scalar
from nominatim_db.tools import refresh
from nominatim_db.db.connection import register_hstore, execute_scalar
from nominatim_db.tokenizer import factory as tokenizer_factory
from steps.utils import run_script
class NominatimEnvironment:
""" Collects all functions for the execution of Nominatim functions.
@@ -62,7 +60,6 @@ class NominatimEnvironment:
dbargs['password'] = self.db_pass
return psycopg.connect(**dbargs)
def write_nominatim_config(self, dbname):
""" Set up a custom test configuration that connects to the given
database. This sets up the environment variables so that they can
@@ -101,7 +98,6 @@ class NominatimEnvironment:
self.website_dir = tempfile.TemporaryDirectory()
def get_test_config(self):
cfg = Configuration(Path(self.website_dir.name), environ=self.test_env)
return cfg
@@ -122,7 +118,6 @@ class NominatimEnvironment:
return dsn
def db_drop_database(self, name):
""" Drop the database with the given name.
"""
@@ -153,13 +148,12 @@ class NominatimEnvironment:
'--osm2pgsql-cache', '1',
'--ignore-errors',
'--offline', '--index-noanalyse')
except:
except: # noqa: E722
self.db_drop_database(self.template_db)
raise
self.run_nominatim('refresh', '--functions')
def setup_api_db(self):
""" Setup a test against the API test database.
"""
@@ -184,13 +178,12 @@ class NominatimEnvironment:
csv_path = str(testdata / 'full_en_phrases_test.csv')
self.run_nominatim('special-phrases', '--import-from-csv', csv_path)
except:
except: # noqa: E722
self.db_drop_database(self.api_test_db)
raise
tokenizer_factory.get_tokenizer_for_db(self.get_test_config())
def setup_unknown_db(self):
""" Setup a test against a non-existing database.
"""
@@ -250,7 +243,6 @@ class NominatimEnvironment:
return False
def reindex_placex(self, db):
""" Run the indexing step until all data in the placex has
been processed. Indexing during updates can produce more data
@@ -259,7 +251,6 @@ class NominatimEnvironment:
"""
self.run_nominatim('index')
def run_nominatim(self, *cmdline):
""" Run the nominatim command-line tool via the library.
"""
@@ -270,7 +261,6 @@ class NominatimEnvironment:
cli_args=cmdline,
environ=self.test_env)
def copy_from_place(self, db):
""" Copy data from place to the placex and location_property_osmline
tables invoking the appropriate triggers.
@@ -293,7 +283,6 @@ class NominatimEnvironment:
and osm_type='W'
and ST_GeometryType(geometry) = 'ST_LineString'""")
def create_api_request_func_starlette(self):
import nominatim_api.server.starlette.server
from asgi_lifespan import LifespanManager
@@ -311,7 +300,6 @@ class NominatimEnvironment:
return _request
def create_api_request_func_falcon(self):
import nominatim_api.server.falcon.server
import falcon.testing
@@ -326,6 +314,3 @@ class NominatimEnvironment:
return response.text, response.status_code
return _request

3
test/bdd/steps/place_inserter.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Helper classes for filling the place table.
@@ -10,6 +10,7 @@ Helper classes for filling the place table.
import random
import string
class PlaceColumn:
""" Helper class to collect contents from a behave table row and
insert it into the place table.

68
test/bdd/steps/steps_api_queries.py
View File

@@ -2,20 +2,16 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2024 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
""" Steps that run queries against the API.
"""
from pathlib import Path
import json
import os
import re
import logging
import asyncio
import xml.etree.ElementTree as ET
from urllib.parse import urlencode
from utils import run_script
from http_responses import GenericResponse, SearchResponse, ReverseResponse, StatusResponse
from check_functions import Bbox, check_for_attributes
from table_compare import NominatimID
@@ -68,7 +64,7 @@ def send_api_query(endpoint, params, fmt, context):
getattr(context, 'http_headers', {})))
@given(u'the HTTP header')
@given('the HTTP header')
def add_http_header(context):
if not hasattr(context, 'http_headers'):
context.http_headers = {}
@@ -77,7 +73,7 @@ def add_http_header(context):
context.http_headers[h] = context.table[0][h]
@when(u'sending (?P<fmt>\S+ )?search query "(?P<query>.*)"(?P<addr> with address)?')
@when(r'sending (?P<fmt>\S+ )?search query "(?P<query>.*)"(?P<addr> with address)?')
def website_search_request(context, fmt, query, addr):
params = {}
if query:
@@ -90,7 +86,7 @@ def website_search_request(context, fmt, query, addr):
context.response = SearchResponse(outp, fmt or 'json', status)
@when('sending v1/reverse at (?P<lat>[\d.-]*),(?P<lon>[\d.-]*)(?: with format (?P<fmt>.+))?')
@when(r'sending v1/reverse at (?P<lat>[\d.-]*),(?P<lon>[\d.-]*)(?: with format (?P<fmt>.+))?')
def api_endpoint_v1_reverse(context, lat, lon, fmt):
params = {}
if lat is not None:
@@ -106,7 +102,7 @@ def api_endpoint_v1_reverse(context, lat, lon, fmt):
context.response = ReverseResponse(outp, fmt or 'xml', status)
@when('sending v1/reverse N(?P<nodeid>\d+)(?: with format (?P<fmt>.+))?')
@when(r'sending v1/reverse N(?P<nodeid>\d+)(?: with format (?P<fmt>.+))?')
def api_endpoint_v1_reverse_from_node(context, nodeid, fmt):
params = {}
params['lon'], params['lat'] = (f'{c:f}' for c in context.osm.grid_node(int(nodeid)))
@@ -115,7 +111,7 @@ def api_endpoint_v1_reverse_from_node(context, nodeid, fmt):
context.response = ReverseResponse(outp, fmt or 'xml', status)
@when(u'sending (?P<fmt>\S+ )?details query for (?P<query>.*)')
@when(r'sending (?P<fmt>\S+ )?details query for (?P<query>.*)')
def website_details_request(context, fmt, query):
params = {}
if query[0] in 'NWR':
@@ -130,38 +126,45 @@ def website_details_request(context, fmt, query):
context.response = GenericResponse(outp, fmt or 'json', status)
@when(u'sending (?P<fmt>\S+ )?lookup query for (?P<query>.*)')
@when(r'sending (?P<fmt>\S+ )?lookup query for (?P<query>.*)')
def website_lookup_request(context, fmt, query):
params = {'osm_ids': query}
outp, status = send_api_query('lookup', params, fmt, context)
context.response = SearchResponse(outp, fmt or 'xml', status)
@when(u'sending (?P<fmt>\S+ )?status query')
@when(r'sending (?P<fmt>\S+ )?status query')
def website_status_request(context, fmt):
params = {}
outp, status = send_api_query('status', params, fmt, context)
context.response = StatusResponse(outp, fmt or 'text', status)
@step(u'(?P<operator>less than|more than|exactly|at least|at most) (?P<number>\d+) results? (?:is|are) returned')
@step(r'(?P<operator>less than|more than|exactly|at least|at most) '
r'(?P<number>\d+) results? (?:is|are) returned')
def validate_result_number(context, operator, number):
context.execute_steps("Then a HTTP 200 is returned")
numres = len(context.response.result)
assert compare(operator, numres, int(number)), \
f"Bad number of results: expected {operator} {number}, got {numres}."
@then(u'a HTTP (?P<status>\d+) is returned')
@then(r'a HTTP (?P<status>\d+) is returned')
def check_http_return_status(context, status):
assert context.response.errorcode == int(status), \
f"Return HTTP status is {context.response.errorcode}."\
f" Full response:\n{context.response.page}"
@then(u'the page contents equals "(?P<text>.+)"')
@then(r'the page contents equals "(?P<text>.+)"')
def check_page_content_equals(context, text):
assert context.response.page == text
@then(u'the result is valid (?P<fmt>\w+)')
@then(r'the result is valid (?P<fmt>\w+)')
def step_impl(context, fmt):
context.execute_steps("Then a HTTP 200 is returned")
if fmt.strip() == 'html':
@@ -178,7 +181,7 @@ def step_impl(context, fmt):
assert context.response.format == fmt
@then(u'a (?P<fmt>\w+) user error is returned')
@then(r'a (?P<fmt>\w+) user error is returned')
def check_page_error(context, fmt):
context.execute_steps("Then a HTTP 400 is returned")
assert context.response.format == fmt
@@ -188,32 +191,34 @@ def check_page_error(context, fmt):
else:
assert re.search(r'({"error":)', context.response.page, re.DOTALL) is not None
@then(u'result header contains')
@then('result header contains')
def check_header_attr(context):
context.execute_steps("Then a HTTP 200 is returned")
for line in context.table:
assert line['attr'] in context.response.header, \
f"Field '{line['attr']}' missing in header. Full header:\n{context.response.header}"
f"Field '{line['attr']}' missing in header. " \
f"Full header:\n{context.response.header}"
value = context.response.header[line['attr']]
assert re.fullmatch(line['value'], value) is not None, \
f"Attribute '{line['attr']}': expected: '{line['value']}', got '{value}'"
@then(u'result header has (?P<neg>not )?attributes (?P<attrs>.*)')
@then('result header has (?P<neg>not )?attributes (?P<attrs>.*)')
def check_header_no_attr(context, neg, attrs):
check_for_attributes(context.response.header, attrs,
'absent' if neg else 'present')
@then(u'results contain(?: in field (?P<field>.*))?')
def step_impl(context, field):
@then(r'results contain(?: in field (?P<field>.*))?')
def results_contain_in_field(context, field):
context.execute_steps("then at least 1 result is returned")
for line in context.table:
context.response.match_row(line, context=context, field=field)
@then(u'result (?P<lid>\d+ )?has (?P<neg>not )?attributes (?P<attrs>.*)')
@then(r'result (?P<lid>\d+ )?has (?P<neg>not )?attributes (?P<attrs>.*)')
def validate_attributes(context, lid, neg, attrs):
for i in make_todo_list(context, lid):
check_for_attributes(context.response.result[i], attrs,
@@ -221,7 +226,7 @@ def validate_attributes(context, lid, neg, attrs):
@then(u'result addresses contain')
def step_impl(context):
def result_addresses_contain(context):
context.execute_steps("then at least 1 result is returned")
for line in context.table:
@@ -231,8 +236,9 @@ def step_impl(context):
if name != 'ID':
context.response.assert_address_field(idx, name, value)
@then(u'address of result (?P<lid>\d+) has(?P<neg> no)? types (?P<attrs>.*)')
def check_address(context, lid, neg, attrs):
@then(r'address of result (?P<lid>\d+) has(?P<neg> no)? types (?P<attrs>.*)')
def check_address_has_types(context, lid, neg, attrs):
context.execute_steps(f"then more than {lid} results are returned")
addr_parts = context.response.result[int(lid)]['address']
@@ -243,7 +249,8 @@ def check_address(context, lid, neg, attrs):
else:
assert attr in addr_parts
@then(u'address of result (?P<lid>\d+) (?P<complete>is|contains)')
@then(r'address of result (?P<lid>\d+) (?P<complete>is|contains)')
def check_address(context, lid, complete):
context.execute_steps(f"then more than {lid} results are returned")
@@ -258,7 +265,7 @@ def check_address(context, lid, complete):
assert len(addr_parts) == 0, f"Additional address parts found: {addr_parts!s}"
@then(u'result (?P<lid>\d+ )?has bounding box in (?P<coords>[\d,.-]+)')
@then(r'result (?P<lid>\d+ )?has bounding box in (?P<coords>[\d,.-]+)')
def check_bounding_box_in_area(context, lid, coords):
expected = Bbox(coords)
@@ -269,7 +276,7 @@ def check_bounding_box_in_area(context, lid, coords):
f"Bbox is not contained in {expected}")
@then(u'result (?P<lid>\d+ )?has centroid in (?P<coords>[\d,.-]+)')
@then(r'result (?P<lid>\d+ )?has centroid in (?P<coords>[\d,.-]+)')
def check_centroid_in_area(context, lid, coords):
expected = Bbox(coords)
@@ -280,7 +287,7 @@ def check_centroid_in_area(context, lid, coords):
f"Centroid is not inside {expected}")
@then(u'there are(?P<neg> no)? duplicates')
@then('there are(?P<neg> no)? duplicates')
def check_for_duplicates(context, neg):
context.execute_steps("then at least 1 result is returned")
@@ -298,4 +305,3 @@ def check_for_duplicates(context, neg):
assert not has_dupe, f"Found duplicate for {dup}"
else:
assert has_dupe, "No duplicates found"

50
test/bdd/steps/steps_db_ops.py
View File

@@ -2,9 +2,8 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2024 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
import logging
from itertools import chain
import psycopg
@@ -13,9 +12,9 @@ from psycopg import sql as pysql
from place_inserter import PlaceColumn
from table_compare import NominatimID, DBRow
from nominatim_db.indexer import indexer
from nominatim_db.tokenizer import factory as tokenizer_factory
def check_database_integrity(context):
""" Check some generic constraints on the tables.
"""
@@ -31,10 +30,9 @@ def check_database_integrity(context):
cur.execute("SELECT count(*) FROM word WHERE word_token = ''")
assert cur.fetchone()[0] == 0, "Empty word tokens found in word table"
# GIVEN ##################################
################################ GIVEN ##################################
@given("the (?P<named>named )?places")
def add_data_to_place_table(context, named):
""" Add entries into the place table. 'named places' makes sure that
@@ -46,6 +44,7 @@ def add_data_to_place_table(context, named):
PlaceColumn(context).add_row(row, named is not None).db_insert(cur)
cur.execute('ALTER TABLE place ENABLE TRIGGER place_before_insert')
@given("the relations")
def add_data_to_planet_relations(context):
""" Add entries into the osm2pgsql relation middle table. This is needed
@@ -77,9 +76,11 @@ def add_data_to_planet_relations(context):
else:
members = None
tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings if h.startswith("tags+")])
tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings
if h.startswith("tags+")])
cur.execute("""INSERT INTO planet_osm_rels (id, way_off, rel_off, parts, members, tags)
cur.execute("""INSERT INTO planet_osm_rels (id, way_off, rel_off,
parts, members, tags)
VALUES (%s, %s, %s, %s, %s, %s)""",
(r['id'], last_node, last_way, parts, members, list(tags)))
else:
@@ -99,6 +100,7 @@ def add_data_to_planet_relations(context):
(r['id'], psycopg.types.json.Json(tags),
psycopg.types.json.Json(members)))
@given("the ways")
def add_data_to_planet_ways(context):
""" Add entries into the osm2pgsql way middle table. This is necessary for
@@ -110,7 +112,8 @@ def add_data_to_planet_ways(context):
json_tags = row is not None and row['value'] != '1'
for r in context.table:
if json_tags:
tags = psycopg.types.json.Json({h[5:]: r[h] for h in r.headings if h.startswith("tags+")})
tags = psycopg.types.json.Json({h[5:]: r[h] for h in r.headings
if h.startswith("tags+")})
else:
tags = list(chain.from_iterable([(h[5:], r[h])
for h in r.headings if h.startswith("tags+")]))
@@ -119,7 +122,8 @@ def add_data_to_planet_ways(context):
cur.execute("INSERT INTO planet_osm_ways (id, nodes, tags) VALUES (%s, %s, %s)",
(r['id'], nodes, tags))
################################ WHEN ##################################
# WHEN ##################################
@when("importing")
def import_and_index_data_from_place_table(context):
@@ -136,6 +140,7 @@ def import_and_index_data_from_place_table(context):
# itself.
context.log_capture.buffer.clear()
@when("updating places")
def update_place_table(context):
""" Update the place table with the given data. Also runs all triggers
@@ -164,6 +169,7 @@ def update_postcodes(context):
"""
context.nominatim.run_nominatim('refresh', '--postcodes')
@when("marking for delete (?P<oids>.*)")
def delete_places(context, oids):
""" Remove entries from the place table. Multiple ids may be given
@@ -184,7 +190,8 @@ def delete_places(context, oids):
# itself.
context.log_capture.buffer.clear()
################################ THEN ##################################
# THEN ##################################
@then("(?P<table>placex|place) contains(?P<exact> exactly)?")
def check_place_contents(context, table, exact):
@@ -201,7 +208,8 @@ def check_place_contents(context, table, exact):
expected_content = set()
for row in context.table:
nid = NominatimID(row['object'])
query = 'SELECT *, ST_AsText(geometry) as geomtxt, ST_GeometryType(geometry) as geometrytype'
query = """SELECT *, ST_AsText(geometry) as geomtxt,
ST_GeometryType(geometry) as geometrytype """
if table == 'placex':
query += ' ,ST_X(centroid) as cx, ST_Y(centroid) as cy'
query += " FROM %s WHERE {}" % (table, )
@@ -261,7 +269,7 @@ def check_search_name_contents(context, exclude):
if not exclude:
assert len(tokens) >= len(items), \
"No word entry found for {}. Entries found: {!s}".format(value, len(tokens))
f"No word entry found for {value}. Entries found: {len(tokens)}"
for word, token, wid in tokens:
if exclude:
assert wid not in res[name], \
@@ -272,6 +280,7 @@ def check_search_name_contents(context, exclude):
elif name != 'object':
assert db_row.contains(name, value), db_row.assert_msg(name, value)
@then("search_name has no entry for (?P<oid>.*)")
def check_search_name_has_entry(context, oid):
""" Check that there is noentry in the search_name table for the given
@@ -283,6 +292,7 @@ def check_search_name_has_entry(context, oid):
assert cur.rowcount == 0, \
"Found {} entries for ID {}".format(cur.rowcount, oid)
@then("location_postcode contains exactly")
def check_location_postcode(context):
""" Check full contents for location_postcode table. Each row represents a table row
@@ -294,7 +304,7 @@ def check_location_postcode(context):
with context.db.cursor() as cur:
cur.execute("SELECT *, ST_AsText(geometry) as geomtxt FROM location_postcode")
assert cur.rowcount == len(list(context.table)), \
"Postcode table has {} rows, expected {}.".format(cur.rowcount, len(list(context.table)))
"Postcode table has {cur.rowcount} rows, expected {len(list(context.table))}."
results = {}
for row in cur:
@@ -309,6 +319,7 @@ def check_location_postcode(context):
db_row.assert_row(row, ('country', 'postcode'))
@then("there are(?P<exclude> no)? word tokens for postcodes (?P<postcodes>.*)")
def check_word_table_for_postcodes(context, exclude, postcodes):
""" Check that the tokenizer produces postcode tokens for the given
@@ -335,6 +346,7 @@ def check_word_table_for_postcodes(context, exclude, postcodes):
assert set(found) == set(plist), \
f"Missing postcodes {set(plist) - set(found)}. Found: {found}"
@then("place_addressline contains")
def check_place_addressline(context):
""" Check the contents of the place_addressline table. Each row represents
@@ -352,11 +364,12 @@ def check_place_addressline(context):
WHERE place_id = %s AND address_place_id = %s""",
(pid, apid))
assert cur.rowcount > 0, \
"No rows found for place %s and address %s" % (row['object'], row['address'])
f"No rows found for place {row['object']} and address {row['address']}."
for res in cur:
DBRow(nid, res, context).assert_row(row, ('address', 'object'))
@then("place_addressline doesn't contain")
def check_place_addressline_exclude(context):
""" Check that the place_addressline doesn't contain any entries for the
@@ -371,9 +384,10 @@ def check_place_addressline_exclude(context):
WHERE place_id = %s AND address_place_id = %s""",
(pid, apid))
assert cur.rowcount == 0, \
"Row found for place %s and address %s" % (row['object'], row['address'])
f"Row found for place {row['object']} and address {row['address']}."
@then("W(?P<oid>\d+) expands to(?P<neg> no)? interpolation")
@then(r"W(?P<oid>\d+) expands to(?P<neg> no)? interpolation")
def check_location_property_osmline(context, oid, neg):
""" Check that the given way is present in the interpolation table.
"""
@@ -402,8 +416,9 @@ def check_location_property_osmline(context, oid, neg):
assert not todo, f"Unmatched lines in table: {list(context.table[i] for i in todo)}"
@then("location_property_osmline contains(?P<exact> exactly)?")
def check_place_contents(context, exact):
def check_osmline_contents(context, exact):
""" Check contents of the interpolation table. Each row represents a table row
and all data must match. Data not present in the expected table, may
be arbitrary. The rows are identified via the 'object' column which must
@@ -447,4 +462,3 @@ def check_place_contents(context, exact):
assert expected_content == actual, \
f"Missing entries: {expected_content - actual}\n" \
f"Not expected in table: {actual - expected_content}"

13
test/bdd/steps/steps_osm_data.py
View File

@@ -14,6 +14,7 @@ from nominatim_db.tools.replication import run_osm2pgsql_updates
from geometry_alias import ALIASES
def get_osm2pgsql_options(nominatim_env, fname, append):
return dict(import_file=fname,
osm2pgsql='osm2pgsql',
@@ -25,8 +26,7 @@ def get_osm2pgsql_options(nominatim_env, fname, append):
flatnode_file='',
tablespaces=dict(slim_data='', slim_index='',
main_data='', main_index=''),
append=append
)
append=append)
def write_opl_file(opl, grid):
@@ -48,6 +48,7 @@ def write_opl_file(opl, grid):
return fd.name
@given('the lua style file')
def lua_style_file(context):
""" Define a custom style file to use for the import.
@@ -90,7 +91,7 @@ def define_node_grid(context, grid_step, origin):
@when(u'loading osm data')
def load_osm_file(context):
"""
Load the given data into a freshly created test data using osm2pgsql.
Load the given data into a freshly created test database using osm2pgsql.
No further indexing is done.
The data is expected as attached text in OPL format.
@@ -102,13 +103,14 @@ def load_osm_file(context):
finally:
os.remove(fname)
### reintroduce the triggers/indexes we've lost by having osm2pgsql set up place again
# reintroduce the triggers/indexes we've lost by having osm2pgsql set up place again
cur = context.db.cursor()
cur.execute("""CREATE TRIGGER place_before_delete BEFORE DELETE ON place
FOR EACH ROW EXECUTE PROCEDURE place_delete()""")
cur.execute("""CREATE TRIGGER place_before_insert BEFORE INSERT ON place
FOR EACH ROW EXECUTE PROCEDURE place_insert()""")
cur.execute("""CREATE UNIQUE INDEX idx_place_osm_unique on place using btree(osm_id,osm_type,class,type)""")
cur.execute("""CREATE UNIQUE INDEX idx_place_osm_unique ON place
USING btree(osm_id,osm_type,class,type)""")
context.db.commit()
@@ -132,6 +134,7 @@ def update_from_osm_file(context):
finally:
os.remove(fname)
@when('indexing')
def index_database(context):
"""

12
test/bdd/steps/table_compare.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Functions to facilitate accessing and comparing the content of DB tables.
@@ -16,6 +16,7 @@ from psycopg import sql as pysql
ID_REGEX = re.compile(r"(?P<typ>[NRW])(?P<oid>\d+)(:(?P<cls>\w+))?")
class NominatimID:
""" Splits a unique identifier for places into its components.
As place_ids cannot be used for testing, we use a unique
@@ -165,12 +166,14 @@ class DBRow:
else:
x, y = self.context.osm.grid_node(int(expected))
return math.isclose(float(x), self.db_row['cx']) and math.isclose(float(y), self.db_row['cy'])
return math.isclose(float(x), self.db_row['cx']) \
and math.isclose(float(y), self.db_row['cy'])
def _has_geometry(self, expected):
geom = self.context.osm.parse_geometry(expected)
with self.context.db.cursor(row_factory=psycopg.rows.tuple_row) as cur:
cur.execute(pysql.SQL("""SELECT ST_Equals(ST_SnapToGrid({}, 0.00001, 0.00001),
cur.execute(pysql.SQL("""
SELECT ST_Equals(ST_SnapToGrid({}, 0.00001, 0.00001),
ST_SnapToGrid(ST_SetSRID({}::geometry, 4326), 0.00001, 0.00001))""")
.format(pysql.SQL(geom),
pysql.Literal(self.db_row['geomtxt'])))
@@ -186,7 +189,8 @@ class DBRow:
else:
msg += " No such column."
return msg + "\nFull DB row: {}".format(json.dumps(dict(self.db_row), indent=4, default=str))
return msg + "\nFull DB row: {}".format(json.dumps(dict(self.db_row),
indent=4, default=str))
def _get_actual(self, name):
if '+' in name:

3
test/bdd/steps/utils.py
View File

@@ -2,7 +2,7 @@
#
# This file is part of Nominatim. (https://nominatim.org)
#
# Copyright (C) 2022 by the Nominatim developer community.
# Copyright (C) 2025 by the Nominatim developer community.
# For a full list of authors see the git log.
"""
Various smaller helps for step execution.
@@ -12,6 +12,7 @@ import subprocess
LOG = logging.getLogger(__name__)
def run_script(cmd, **kwargs):
""" Run the given command, check that it is successful and output
when necessary.