bdd tests: get rid of dependency on build path

BDD tests will now use whatever osm2pgsql they find in the PATH.
For testing against legacy tokenizer, use -DSERVER_MODULE_PATH
to point to the module. This will no longer work out of the box.

docs/develop/Development-Environment.md

@@ -50,6 +50,10 @@ The documentation is built with mkdocs:
 * [mkdocstrings](https://mkdocstrings.github.io/) >= 0.25
 * [mkdocs-material](https://squidfunk.github.io/mkdocs-material/)
 
+Please be aware that tests always run against the globally installed
+osm2pgsql, so you need to have this set up. If you want to test against
+the vendored version of osm2pgsql, you need to set the PATH accordingly.
+
 ### Installing prerequisites on Ubuntu/Debian
 
 The Python tools should always be run with the most recent version.

docs/develop/Testing.md

@@ -78,7 +78,6 @@ To run the functional tests, do
 
 The tests can be configured with a set of environment variables (`behave -D key=val`):
 
- * `BUILDDIR` - build directory of Nominatim installation to test
  * `TEMPLATE_DB` - name of template database used as a skeleton for
                    the test databases (db tests)
  * `TEST_DB` - name of test database (db tests)
@@ -91,7 +90,7 @@ The tests can be configured with a set of environment variables (`behave -D key=
  * `DB_USER` - (optional) username of database login
  * `DB_PASS` - (optional) password for database login
  * `SERVER_MODULE_PATH` - (optional) path on the Postgres server to Nominatim
-                          module shared library file
+                          module shared library file (only needed for legacy tokenizer)
  * `REMOVE_TEMPLATE` - if true, the template and API database will not be reused
                        during the next run. Reusing the base templates speeds
                        up tests considerably but might lead to outdated errors

test/bdd/environment.py

@@ -5,16 +5,18 @@
 # Copyright (C) 2024 by the Nominatim developer community.
 # For a full list of authors see the git log.
 from pathlib import Path
+import sys
 
 from behave import *
 
+sys.path.insert(1, str(Path(__file__, '..', '..', '..', 'src').resolve()))
+
 from steps.geometry_factory import GeometryFactory
 from steps.nominatim_environment import NominatimEnvironment
 
-TEST_BASE_DIR = Path(__file__) / '..' / '..'
+TEST_BASE_DIR = Path(__file__, '..', '..').resolve()
 
 userconfig = {
-    'BUILDDIR' : (TEST_BASE_DIR / '..' / 'build').resolve(),
     'REMOVE_TEMPLATE' : False,
     'KEEP_TEST_DB' : False,
     'DB_HOST' : None,
@@ -24,7 +26,7 @@ userconfig = {
     'TEMPLATE_DB' : 'test_template_nominatim',
     'TEST_DB' : 'test_nominatim',
     'API_TEST_DB' : 'test_api_nominatim',
-    'API_TEST_FILE'  : (TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf').resolve(),
+    'API_TEST_FILE'  : TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf',
     'SERVER_MODULE_PATH' : None,
     'TOKENIZER' : None, # Test with a custom tokenizer
     'STYLE' : 'extratags',

test/bdd/steps/nominatim_environment.py

@@ -6,14 +6,11 @@
 # For a full list of authors see the git log.
 from pathlib import Path
 import importlib
-import sys
 import tempfile
 
 import psycopg
 from psycopg import sql as pysql
 
-sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..'/ 'src').resolve()))
-
 from nominatim_db import cli
 from nominatim_db.config import Configuration
 from nominatim_db.db.connection import Connection, register_hstore, execute_scalar
@@ -26,7 +23,6 @@ class NominatimEnvironment:
     """
 
     def __init__(self, config):
-        self.build_dir = Path(config['BUILDDIR']).resolve()
         self.src_dir = (Path(__file__) / '..' / '..' / '..' / '..').resolve()
         self.db_host = config['DB_HOST']
         self.db_port = config['DB_PORT']
@@ -56,6 +52,9 @@ class NominatimEnvironment:
                 raise RuntimeError(f"Unknown API engine '{config['API_ENGINE']}'")
             self.api_engine = getattr(self, f"create_api_request_func_{config['API_ENGINE']}")()
 
+        if self.tokenizer == 'legacy' and self.server_module_path is None:
+            raise RuntimeError("You must set -DSERVER_MODULE_PATH when testing the legacy tokenizer.")
+
     def connect_database(self, dbname):
         """ Return a connection to the database with the given name.
             Uses configured host, user and port.
@@ -107,8 +106,6 @@ class NominatimEnvironment:
         self.test_env['NOMINATIM_DATADIR'] = str((self.src_dir / 'data').resolve())
         self.test_env['NOMINATIM_SQLDIR'] = str((self.src_dir / 'lib-sql').resolve())
         self.test_env['NOMINATIM_CONFIGDIR'] = str((self.src_dir / 'settings').resolve())
-        self.test_env['NOMINATIM_DATABASE_MODULE_SRC_PATH'] = str((self.build_dir / 'module').resolve())
-        self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = str((self.build_dir / 'osm2pgsql' / 'osm2pgsql').resolve())
         if self.tokenizer is not None:
             self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer
         if self.import_style is not None:
@@ -116,9 +113,6 @@ class NominatimEnvironment:
 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
-        else:
-            # avoid module being copied into the temporary environment
-            self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = str((self.build_dir / 'module').resolve())
 
         if self.website_dir is not None:
             self.website_dir.cleanup()
@@ -137,8 +131,7 @@ class NominatimEnvironment:
 
     def get_test_config(self):
         cfg = Configuration(Path(self.website_dir.name), environ=self.test_env)
-        cfg.set_libdirs(module=self.build_dir / 'module',
+        cfg.set_libdirs(module=self.server_module_path)
-                        osm2pgsql=self.build_dir / 'osm2pgsql' / 'osm2pgsql')
         return cfg
 
     def get_libpq_dsn(self):
@@ -305,8 +298,8 @@ class NominatimEnvironment:
         if self.website_dir is not None:
             cmdline = list(cmdline) + ['--project-dir', self.website_dir.name]
 
-        cli.nominatim(module_dir='',
+        cli.nominatim(module_dir=self.server_module_path,
-                      osm2pgsql_path=str(self.build_dir / 'osm2pgsql' / 'osm2pgsql'),
+                      osm2pgsql_path=None,
                       cli_args=cmdline,
                       environ=self.test_env)
 

test/bdd/steps/steps_osm_data.py

@@ -16,7 +16,7 @@ from geometry_alias import ALIASES
 
 def get_osm2pgsql_options(nominatim_env, fname, append):
     return dict(import_file=fname,
-                osm2pgsql=str(nominatim_env.build_dir / 'osm2pgsql' / 'osm2pgsql'),
+                osm2pgsql='osm2pgsql',
                 osm2pgsql_cache=50,
                 osm2pgsql_style=str(nominatim_env.get_test_config().get_import_style_file()),
                 osm2pgsql_style_path=nominatim_env.get_test_config().config_dir,