diff --git a/test/python/conftest.py b/test/python/conftest.py index 8ba27b34..0234dd05 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -22,7 +22,6 @@ from nominatim_db.db.sql_preprocessor import SQLPreprocessor import nominatim_db.tokenizer.factory import dummy_tokenizer -import mocks from cursor import CursorForTesting @@ -243,10 +242,37 @@ def place_postcode_row(place_postcode_table, temp_db_cursor): @pytest.fixture -def placex_table(temp_db_with_extensions, temp_db_conn): - """ Create an empty version of the place table. +def placex_table(temp_db_with_extensions, temp_db_conn, load_sql, place_table): + """ Create an empty version of the placex table. """ - return mocks.MockPlacexTable(temp_db_conn) + load_sql('tables/placex.sql') + temp_db_conn.execute("CREATE SEQUENCE IF NOT EXISTS seq_place START 1") + + +@pytest.fixture +def placex_row(placex_table, temp_db_cursor): + """ A factory for rows in the placex table. The table is created as a + prerequisite to the fixture. + """ + idseq = itertools.count(1001) + + def _add(osm_type='N', osm_id=None, cls='amenity', typ='cafe', names=None, + admin_level=None, address=None, extratags=None, geom='POINT(10 4)', + country=None, housenumber=None, rank_search=30, rank_address=30, + centroid='POINT(10 4)', indexed_status=0, indexed_date=None): + args = {'place_id': pysql.SQL("nextval('seq_place')"), + 'osm_type': osm_type, 'osm_id': osm_id or next(idseq), + 'class': cls, 'type': typ, 'name': names, 'admin_level': admin_level, + 'address': address, 'housenumber': housenumber, + 'rank_search': rank_search, 'rank_address': rank_address, + 'extratags': extratags, + 'centroid': _with_srid(centroid), 'geometry': _with_srid(geom), + 'country_code': country, + 'indexed_status': indexed_status, 'indexed_date': indexed_date, + 'partition': pysql.Literal(0), 'geometry_sector': pysql.Literal(1)} + return temp_db_cursor.insert_row('placex', **args) + + return _add @pytest.fixture diff --git a/test/python/mocks.py b/test/python/mocks.py deleted file mode 100644 index 7cc8c815..00000000 --- a/test/python/mocks.py +++ /dev/null @@ -1,66 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later -# -# This file is part of Nominatim. (https://nominatim.org) -# -# Copyright (C) 2025 by the Nominatim developer community. -# For a full list of authors see the git log. -""" -Custom mocks for testing. -""" -import itertools - - -class MockPlacexTable: - """ A placex table for testing. - """ - def __init__(self, conn): - self.idseq = itertools.count(10000) - self.conn = conn - with conn.cursor() as cur: - cur.execute("""CREATE TABLE placex ( - place_id BIGINT, - parent_place_id BIGINT, - linked_place_id BIGINT, - importance FLOAT, - indexed_date TIMESTAMP, - geometry_sector INTEGER, - rank_address SMALLINT, - rank_search SMALLINT, - partition SMALLINT, - indexed_status SMALLINT, - osm_id int8, - osm_type char(1), - class text, - type text, - name hstore, - admin_level smallint, - address hstore, - extratags hstore, - token_info jsonb, - geometry Geometry(Geometry,4326), - wikipedia TEXT, - country_code varchar(2), - housenumber TEXT, - postcode TEXT, - centroid GEOMETRY(Geometry, 4326))""") - cur.execute("CREATE SEQUENCE IF NOT EXISTS seq_place") - conn.commit() - - def add(self, osm_type='N', osm_id=None, cls='amenity', typ='cafe', names=None, - admin_level=None, address=None, extratags=None, geom='POINT(10 4)', - country=None, housenumber=None, rank_search=30, centroid=None): - with self.conn.cursor() as cur: - cur.execute("""INSERT INTO placex (place_id, osm_type, osm_id, class, - type, name, admin_level, address, - housenumber, rank_search, - extratags, centroid, geometry, country_code) - VALUES(nextval('seq_place'), %s, %s, %s, %s, %s, %s, - %s, %s, %s, %s, %s, %s, %s) - RETURNING place_id""", - (osm_type, osm_id or next(self.idseq), cls, typ, names, - admin_level, address, housenumber, rank_search, - extratags, centroid, 'SRID=4326;' + geom, - country)) - place_id = cur.fetchone()[0] - self.conn.commit() - return place_id diff --git a/test/python/tokenizer/test_icu.py b/test/python/tokenizer/test_icu.py index 67c2b84b..122c10c3 100644 --- a/test/python/tokenizer/test_icu.py +++ b/test/python/tokenizer/test_icu.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for ICU tokenizer. @@ -653,22 +653,21 @@ class TestUpdateWordTokens: self.tok.update_word_tokens() assert word_table.count_housenumbers() == 1 - def test_keep_housenumbers_from_placex_table(self, add_housenumber, word_table, - placex_table): + def test_keep_housenumbers_from_placex_table(self, add_housenumber, word_table, placex_row): add_housenumber(9999, '5432a') add_housenumber(9990, '34z') - placex_table.add(housenumber='34z') - placex_table.add(housenumber='25432a') + placex_row(housenumber='34z') + placex_row(housenumber='25432a') assert word_table.count_housenumbers() == 2 self.tok.update_word_tokens() assert word_table.count_housenumbers() == 1 def test_keep_housenumbers_from_placex_table_hnr_list(self, add_housenumber, - word_table, placex_table): + word_table, placex_row): add_housenumber(9991, '9 b') add_housenumber(9990, '34z') - placex_table.add(housenumber='9 a;9 b;9 c') + placex_row(housenumber='9 a;9 b;9 c') assert word_table.count_housenumbers() == 2 self.tok.update_word_tokens() diff --git a/test/python/tools/test_admin.py b/test/python/tools/test_admin.py index a3783f1a..788c59b1 100644 --- a/test/python/tools/test_admin.py +++ b/test/python/tools/test_admin.py @@ -8,6 +8,7 @@ Tests for maintenance and analysis functions. """ import pytest +import datetime as dt from nominatim_db.errors import UsageError from nominatim_db.tools import admin @@ -61,15 +62,14 @@ def test_analyse_indexing_unknown_osmid(project_env): admin.analyse_indexing(project_env, osm_id='W12345674') -def test_analyse_indexing_with_place_id(project_env, temp_db_cursor): - temp_db_cursor.execute("INSERT INTO placex (place_id) VALUES(12345)") +def test_analyse_indexing_with_place_id(project_env, placex_row): + place_id = placex_row() - admin.analyse_indexing(project_env, place_id=12345) + admin.analyse_indexing(project_env, place_id=place_id) -def test_analyse_indexing_with_osm_id(project_env, temp_db_cursor): - temp_db_cursor.execute("""INSERT INTO placex (place_id, osm_type, osm_id) - VALUES(9988, 'N', 10000)""") +def test_analyse_indexing_with_osm_id(project_env, placex_row): + placex_row(osm_type='N', osm_id=10000) admin.analyse_indexing(project_env, osm_id='N10000') @@ -77,7 +77,7 @@ def test_analyse_indexing_with_osm_id(project_env, temp_db_cursor): class TestAdminCleanDeleted: @pytest.fixture(autouse=True) - def setup_polygon_delete(self, project_env, table_factory, country_table, place_table, + def setup_polygon_delete(self, project_env, table_factory, place_table, placex_row, osmline_table, temp_db_cursor, temp_db_conn, def_config, src_dir): """ Set up place_force_delete function and related tables """ @@ -91,12 +91,15 @@ class TestAdminCleanDeleted: ((100, 'N', 'boundary', 'administrative'), (145, 'N', 'boundary', 'administrative'), (175, 'R', 'landcover', 'grass'))) - temp_db_cursor.execute(""" - INSERT INTO placex (place_id, osm_id, osm_type, class, type, - indexed_date, indexed_status) - VALUES(1, 100, 'N', 'boundary', 'administrative', current_date - INTERVAL '1 month', 1), - (2, 145, 'N', 'boundary', 'administrative', current_date - INTERVAL '3 month', 1), - (3, 175, 'R', 'landcover', 'grass', current_date - INTERVAL '3 months', 1)""") + + now = dt.datetime.now() + placex_row(osm_type='N', osm_id=100, cls='boundary', typ='administrative', + indexed_status=1, indexed_date=now - dt.timedelta(days=30)) + placex_row(osm_type='N', osm_id=145, cls='boundary', typ='administrative', + indexed_status=1, indexed_date=now - dt.timedelta(days=90)) + placex_row(osm_type='R', osm_id=175, cls='landcover', typ='grass', + indexed_status=1, indexed_date=now - dt.timedelta(days=90)) + # set up tables and triggers for utils function table_factory('place_to_be_deleted', """osm_id BIGINT, diff --git a/test/python/tools/test_database_import.py b/test/python/tools/test_database_import.py index f901067b..ec8f504c 100644 --- a/test/python/tools/test_database_import.py +++ b/test/python/tools/test_database_import.py @@ -176,6 +176,18 @@ async def test_load_data(dsn, place_row, placex_table, osmline_table, geom='LINESTRING(0 0, 10 10)') temp_db_cursor.execute(""" + CREATE OR REPLACE FUNCTION placex_insert() RETURNS TRIGGER AS $$ + BEGIN + NEW.place_id := nextval('seq_place'); + NEW.indexed_status := 1; + NEW.centroid := ST_Centroid(NEW.geometry); + NEW.partition := 0; + NEW.geometry_sector := 2424; + NEW.rank_address := 30; + NEW.rank_search := 30; + RETURN NEW; + END; $$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + CREATE OR REPLACE FUNCTION osmline_insert() RETURNS TRIGGER AS $$ BEGIN NEW.place_id := nextval('seq_place'); @@ -187,8 +199,11 @@ async def test_load_data(dsn, place_row, placex_table, osmline_table, RETURN NEW; END; $$ LANGUAGE plpgsql STABLE PARALLEL SAFE; + CREATE TRIGGER placex_before_insert BEFORE INSERT ON placex + FOR EACH ROW EXECUTE PROCEDURE placex_insert(); + CREATE TRIGGER osmline_before_insert BEFORE INSERT ON location_property_osmline - FOR EACH ROW EXECUTE PROCEDURE osmline_insert() + FOR EACH ROW EXECUTE PROCEDURE osmline_insert(); """) await database_import.load_data(dsn, threads) diff --git a/test/python/tools/test_import_special_phrases.py b/test/python/tools/test_import_special_phrases.py index db2d02e2..acacc480 100644 --- a/test/python/tools/test_import_special_phrases.py +++ b/test/python/tools/test_import_special_phrases.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for import special phrases methods @@ -125,9 +125,8 @@ def test_grant_access_to_web_user(temp_db_conn, temp_db_cursor, table_factory, phrase_class, phrase_type) -def test_create_place_classtype_table_and_indexes( - temp_db_cursor, def_config, placex_table, - sp_importer, temp_db_conn, monkeypatch): +def test_create_place_classtype_table_and_indexes(temp_db_cursor, def_config, placex_row, + sp_importer, temp_db_conn, monkeypatch): """ Test that _create_place_classtype_table_and_indexes() create the right place_classtype tables and place_id indexes @@ -136,7 +135,7 @@ def test_create_place_classtype_table_and_indexes( """ pairs = set([('class1', 'type1'), ('class2', 'type2')]) for pair in pairs: - placex_table.add(cls=pair[0], typ=pair[1]) # adding to db + placex_row(cls=pair[0], typ=pair[1]) # adding to db sp_importer._create_classtype_table_and_indexes(pairs) temp_db_conn.commit() @@ -178,7 +177,7 @@ def test_remove_non_existent_tables_from_db(sp_importer, default_phrases, @pytest.mark.parametrize("should_replace", [(True), (False)]) def test_import_phrases(monkeypatch, temp_db_cursor, def_config, sp_importer, - placex_table, table_factory, tokenizer_mock, + placex_row, table_factory, tokenizer_mock, xml_wiki_content, should_replace): """ Check that the main import_phrases() method is well executed. @@ -199,8 +198,8 @@ def test_import_phrases(monkeypatch, temp_db_cursor, def_config, sp_importer, type_test = 'zip_line' tokenizer = tokenizer_mock() - placex_table.add(cls=class_test, typ=type_test) # in db for special phrase filtering - placex_table.add(cls='amenity', typ='animal_shelter') # in db for special phrase filtering + placex_row(cls=class_test, typ=type_test) # in db for special phrase filtering + placex_row(cls='amenity', typ='animal_shelter') # in db for special phrase filtering sp_importer.import_phrases(tokenizer, should_replace) assert len(tokenizer.analyser_cache['special_phrases']) == 19 @@ -257,7 +256,7 @@ def check_placeid_and_centroid_indexes(temp_db_cursor, phrase_class, phrase_type @pytest.mark.parametrize("should_replace", [(True), (False)]) def test_import_phrases_special_phrase_filtering(monkeypatch, temp_db_cursor, def_config, - sp_importer, placex_table, tokenizer_mock, + sp_importer, placex_row, tokenizer_mock, xml_wiki_content, should_replace): monkeypatch.setattr('nominatim_db.tools.special_phrases.sp_wiki_loader._get_wiki_content', @@ -266,7 +265,7 @@ def test_import_phrases_special_phrase_filtering(monkeypatch, temp_db_cursor, de class_test = 'aerialway' type_test = 'zip_line' - placex_table.add(cls=class_test, typ=type_test) # add to the database to make valid + placex_row(cls=class_test, typ=type_test) # add to the database to make valid tokenizer = tokenizer_mock() sp_importer.import_phrases(tokenizer, should_replace) @@ -276,11 +275,11 @@ def test_import_phrases_special_phrase_filtering(monkeypatch, temp_db_cursor, de assert check_grant_access(temp_db_cursor, def_config.DATABASE_WEBUSER, class_test, type_test) -def test_get_classtype_pairs_directly(placex_table, temp_db_conn, sp_importer): +def test_get_classtype_pairs_directly(placex_row, temp_db_conn, sp_importer): for _ in range(101): - placex_table.add(cls='highway', typ='residential') + placex_row(cls='highway', typ='residential') for _ in range(99): - placex_table.add(cls='amenity', typ='toilet') + placex_row(cls='amenity', typ='toilet') temp_db_conn.commit() diff --git a/test/python/tools/test_postcodes.py b/test/python/tools/test_postcodes.py index 02cd2a1a..c63e0d21 100644 --- a/test/python/tools/test_postcodes.py +++ b/test/python/tools/test_postcodes.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for functions to maintain the artificial postcode table. @@ -75,20 +75,18 @@ class MockPostcodeTable: @pytest.fixture def postcode_table(def_config, temp_db_conn, placex_table, table_factory): country_info.setup_country_config(def_config) - table_factory('country_name', 'partition INT', ((0, ), (1, ), (2, ))) return MockPostcodeTable(temp_db_conn, def_config) @pytest.fixture -def insert_implicit_postcode(placex_table, place_postcode_row): +def insert_implicit_postcode(placex_row, place_postcode_row): """ Insert data into the placex and place table which can then be used to compute one postcode. """ def _insert_implicit_postcode(osm_id, country, geometry, postcode, in_placex=False): if in_placex: - placex_table.add(osm_id=osm_id, country=country, geom=geometry, - centroid=f'SRID=4326;{geometry}', - address={'postcode': postcode}) + placex_row(osm_id=osm_id, country=country, geom=geometry, + centroid=geometry, address={'postcode': postcode}) else: place_postcode_row(osm_id=osm_id, centroid=geometry, country=country, postcode=postcode) diff --git a/test/python/tools/test_refresh.py b/test/python/tools/test_refresh.py index 91903e29..aa6473b4 100644 --- a/test/python/tools/test_refresh.py +++ b/test/python/tools/test_refresh.py @@ -42,8 +42,8 @@ def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, r @pytest.mark.parametrize('osm_type', ('N', 'W', 'R')) -def test_invalidate_osm_object_simple(placex_table, osm_type, temp_db_conn, temp_db_cursor): - placex_table.add(osm_type=osm_type, osm_id=57283) +def test_invalidate_osm_object_simple(placex_row, osm_type, temp_db_conn, temp_db_cursor): + placex_row(osm_type=osm_type, osm_id=57283) refresh.invalidate_osm_object(osm_type, 57283, temp_db_conn, recursive=False) temp_db_conn.commit() @@ -53,8 +53,8 @@ def test_invalidate_osm_object_simple(placex_table, osm_type, temp_db_conn, temp (osm_type, 57283)) -def test_invalidate_osm_object_nonexisting_simple(placex_table, temp_db_conn, temp_db_cursor): - placex_table.add(osm_type='W', osm_id=57283) +def test_invalidate_osm_object_nonexisting_simple(placex_row, temp_db_conn, temp_db_cursor): + placex_row(osm_type='W', osm_id=57283) refresh.invalidate_osm_object('N', 57283, temp_db_conn, recursive=False) temp_db_conn.commit() @@ -64,8 +64,8 @@ def test_invalidate_osm_object_nonexisting_simple(placex_table, temp_db_conn, te @pytest.mark.parametrize('osm_type', ('N', 'W', 'R')) -def test_invalidate_osm_object_recursive(placex_table, osm_type, temp_db_conn, temp_db_cursor): - placex_table.add(osm_type=osm_type, osm_id=57283) +def test_invalidate_osm_object_recursive(placex_row, osm_type, temp_db_conn, temp_db_cursor): + placex_row(osm_type=osm_type, osm_id=57283) temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION place_force_update(placeid BIGINT) RETURNS BOOLEAN AS $$ diff --git a/test/python/tools/test_refresh_wiki_data.py b/test/python/tools/test_refresh_wiki_data.py index 6d4a727d..3de37647 100644 --- a/test/python/tools/test_refresh_wiki_data.py +++ b/test/python/tools/test_refresh_wiki_data.py @@ -2,7 +2,7 @@ # # This file is part of Nominatim. (https://nominatim.org) # -# Copyright (C) 2025 by the Nominatim developer community. +# Copyright (C) 2026 by the Nominatim developer community. # For a full list of authors see the git log. """ Tests for correctly assigning wikipedia pages to places. @@ -38,7 +38,7 @@ def wiki_csv(tmp_path, sql_preprocessor): {'wikipedia': 'en:Test'}, {'wikidata': 'Q123'}]) def test_wikipedia(dsn, temp_db_conn, temp_db_cursor, table_factory, - def_config, wiki_csv, placex_table, extra): + def_config, wiki_csv, placex_row, extra): import_wikipedia_articles(dsn, wiki_csv([('en', 'Test', 0.3, 'Q123')])) create_functions(temp_db_conn, def_config) @@ -46,7 +46,7 @@ def test_wikipedia(dsn, temp_db_conn, temp_db_cursor, table_factory, 'SELECT language, title, importance, wikidata FROM wikimedia_importance') assert content == set([('en', 'Test', 0.3, 'Q123')]) - place_id = placex_table.add(osm_id=12, extratags=extra) + place_id = placex_row(osm_id=12, extratags=extra) table_factory('search_name', 'place_id BIGINT, importance FLOAT', [(place_id, 0.2)]) @@ -61,11 +61,11 @@ def test_wikipedia(dsn, temp_db_conn, temp_db_cursor, table_factory, def test_wikipedia_no_match(dsn, temp_db_conn, temp_db_cursor, def_config, wiki_csv, - placex_table, table_factory): + placex_row, table_factory): import_wikipedia_articles(dsn, wiki_csv([('de', 'Test', 0.3, 'Q123')])) create_functions(temp_db_conn, def_config) - place_id = placex_table.add(osm_id=12, extratags={'wikipedia': 'en:Test'}, rank_search=10) + place_id = placex_row(osm_id=12, extratags={'wikipedia': 'en:Test'}, rank_search=10) table_factory('search_name', 'place_id BIGINT, importance FLOAT', [(place_id, 0.2)]) diff --git a/test/python/tools/test_sp_importer.py b/test/python/tools/test_sp_importer.py index c64c2b7d..ba780e0d 100644 --- a/test/python/tools/test_sp_importer.py +++ b/test/python/tools/test_sp_importer.py @@ -1,69 +1,70 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2026 by the Nominatim developer community. +# For a full list of authors see the git log. + from nominatim_db.tools.special_phrases.sp_importer import SPImporter # Testing Database Class Pair Retrival using Conftest.py and placex -def test_get_classtype_pair_data(placex_table, def_config, temp_db_conn): +def test_get_classtype_pair_data(placex_row, def_config, temp_db_conn): for _ in range(100): - placex_table.add(cls='highway', typ='motorway') # edge case 100 + placex_row(cls='highway', typ='motorway') # edge case 100 for _ in range(99): - placex_table.add(cls='amenity', typ='prison') # edge case 99 + placex_row(cls='amenity', typ='prison') # edge case 99 for _ in range(150): - placex_table.add(cls='tourism', typ='hotel') + placex_row(cls='tourism', typ='hotel') importer = SPImporter(config=def_config, conn=temp_db_conn, sp_loader=None) result = importer.get_classtype_pairs(min=100) - expected = { + assert result == { ("highway", "motorway"), ("tourism", "hotel") } - assert result == expected, f"Expected {expected}, got {result}" - -def test_get_classtype_pair_data_more(placex_table, def_config, temp_db_conn): +def test_get_classtype_pair_data_more(placex_row, def_config, temp_db_conn): for _ in range(99): - placex_table.add(cls='emergency', typ='firehydrant') # edge case 99, not included + placex_row(cls='emergency', typ='firehydrant') # edge case 99, not included for _ in range(199): - placex_table.add(cls='amenity', typ='prison') + placex_row(cls='amenity', typ='prison') for _ in range(3478): - placex_table.add(cls='tourism', typ='hotel') + placex_row(cls='tourism', typ='hotel') importer = SPImporter(config=def_config, conn=temp_db_conn, sp_loader=None) result = importer.get_classtype_pairs(min=100) - expected = { + assert result == { ("amenity", "prison"), ("tourism", "hotel") } - assert result == expected, f"Expected {expected}, got {result}" - -def test_get_classtype_pair_data_default(placex_table, def_config, temp_db_conn): +def test_get_classtype_pair_data_default(placex_row, def_config, temp_db_conn): for _ in range(1): - placex_table.add(cls='emergency', typ='firehydrant') + placex_row(cls='emergency', typ='firehydrant') for _ in range(199): - placex_table.add(cls='amenity', typ='prison') + placex_row(cls='amenity', typ='prison') for _ in range(3478): - placex_table.add(cls='tourism', typ='hotel') + placex_row(cls='tourism', typ='hotel') importer = SPImporter(config=def_config, conn=temp_db_conn, sp_loader=None) result = importer.get_classtype_pairs() - expected = { + assert result == { ("amenity", "prison"), ("tourism", "hotel"), ("emergency", "firehydrant") } - - assert result == expected, f"Expected {expected}, got {result}"