Merge pull request #3509 from lonvia/deprecations

Add deprecation warnings and check for osm2pgsql version

CMakeLists.txt

@@ -271,3 +271,7 @@ if (INSTALL_MUNIN_PLUGINS)
                   munin/nominatim_requests
             DESTINATION ${NOMINATIM_MUNINDIR})
 endif()
+
+message(WARNING "Building with CMake is deprecated and will be removed in Nominatim 5.0."
+                "Use Nominatim pip packages instead.\n"
+                "See https://nominatim.org/release-docs/develop/admin/Installation/#downloading-and-building-nominatim")

docs/admin/Deployment-PHP.md

@@ -1,5 +1,8 @@
 # Deploying Nominatim using the PHP frontend
 
+!!! danger
+    The PHP frontend is deprecated and will be removed in Nominatim 5.0.
+
 The Nominatim API is implemented as a PHP application. The `website/` directory
 in the project directory contains the configured website. You can serve this
 in a production environment with any web server that is capable to run

docs/admin/Import.md

@@ -302,6 +302,9 @@ To run Nominatim via webservers like Apache or nginx, please continue reading
 
 #### Testing the PHP frontend
 
+!!! danger
+    The PHP fronted is deprecated and will be removed in Nominatim 5.0.
+
 You can run a small test server with the PHP frontend like this:
 
 ```sh

docs/admin/Installation.md

@@ -72,7 +72,7 @@ For running the Python frontend:
     * [starlette](https://www.starlette.io/)
   * [uvicorn](https://www.uvicorn.org/)
 
-For running the legacy PHP frontend:
+For running the legacy PHP frontend (deprecated, will be removed in Nominatim 5.0):
 
   * [PHP](https://php.net) (7.3+)
   * PHP-pgsql
@@ -194,6 +194,7 @@ sudo make install
     cmake: `cmake -DBUILD_MODULE=on ../Nominatim`. To compile the module
     you need to have the server development headers for PostgreSQL installed.
     On Ubuntu/Debian run: `sudo apt install postgresql-server-dev-<postgresql version>`
+    The legacy tokenizer is deprecated and will be removed in Nominatim 5.0
 
 
 Nominatim installs itself into `/usr/local` per default. To choose a different

docs/admin/Migration.md

@@ -1,12 +1,17 @@
 # Database Migrations
 
-Since version 3.7.0 Nominatim offers automatic migrations. Please follow
+Nominatim offers automatic migrations since version 3.7. Please follow
 the following steps:
 
-* stop any updates that are potentially running
+* Stop any updates that are potentially running
-* update Nominatim to the newer version
+* Update the backend: `pip install -U nominatim-db`
-* go to your project directory and run `nominatim admin --migrate`
+* Go to your project directory and run `nominatim admin --migrate`
-* (optionally) restart updates
+* Update the frontend: `pip install -U nominatim-api`
+* (optionally) Restart updates
+
+If you are still using CMake for the installation of Nominatim, then you
+need to update the software in one step before migrating the database.
+It is not recommended to do this while the machine is serving requests.
 
 Below you find additional migrations and hints about other structural and
 breaking changes. **Please read them before running the migration.**

docs/customize/SQLite.md

@@ -12,11 +12,16 @@ To use a SQLite database, you need to install:
 
 * SQLite (>= 3.30)
 * Spatialite (> 5.0.0)
+* aiosqlite
 
 On Ubuntu/Debian, you can run:
 
     sudo apt install sqlite3 libsqlite3-mod-spatialite libspatialite7
 
+Install the aiosqlite Python package in your virtual environment:
+
+    /srv/nominatim-venv/bin/pip install aiosqlite
+
 ## Creating a new SQLite database
 
 Nominatim cannot import directly into SQLite database. Instead you have to

docs/customize/Tokenizers.md

@@ -17,6 +17,11 @@ they can be configured.
 
 ## Legacy tokenizer
 
+!!! danger
+    The Legacy tokenizer is deprecated and will be removed in Nominatim 5.0.
+    If you still use a database with the legacy tokenizer, you must reimport
+    it using the ICU tokenizer below.
+
 The legacy tokenizer implements the analysis algorithms of older Nominatim
 versions. It uses a special Postgresql module to normalize names and queries.
 This tokenizer is automatically installed and used when upgrading an older

src/nominatim_db/cli.py

@@ -119,7 +119,13 @@ class CommandlineParser:
         log.warning('Using project directory: %s', str(args.project_dir))
 
         try:
-            return args.command.run(args)
+            ret = args.command.run(args)
+
+            if args.config.TOKENIZER == 'legacy':
+                log.warning('WARNING: the "legacy" tokenizer is deprecated '
+                            'and will be removed in Nominatim 5.0.')
+
+            return ret
         except UsageError as exception:
             if log.isEnabledFor(logging.DEBUG):
                 raise # use Python's exception printing
@@ -169,6 +175,8 @@ class AdminServe:
         if args.engine == 'php':
             if args.config.lib_dir.php is None:
                 raise UsageError("PHP frontend not configured.")
+            LOG.warning('\n\nWARNING: the PHP frontend is deprecated '
+                        'and will be removed in Nominatim 5.0.\n\n')
             run_php_server(args.server, args.project_dir / 'website')
         else:
             asyncio.run(self.run_uvicorn(args))

src/nominatim_db/tokenizer/legacy_tokenizer.py

@@ -38,10 +38,12 @@ LOG = logging.getLogger()
 def create(dsn: str, data_dir: Path) -> 'LegacyTokenizer':
     """ Create a new instance of the tokenizer provided by this module.
     """
+    LOG.warning('WARNING: the legacy tokenizer is deprecated '
+                'and will be removed in Nominatim 5.0.')
     return LegacyTokenizer(dsn, data_dir)
 
 
-def _install_module(config_module_path: str, src_dir: Path, module_dir: Path) -> str:
+def _install_module(config_module_path: str, src_dir: Optional[Path], module_dir: Path) -> str:
     """ Copies the PostgreSQL normalisation module into the project
         directory if necessary. For historical reasons the module is
         saved in the '/module' subdirectory and not with the other tokenizer
@@ -55,6 +57,10 @@ def _install_module(config_module_path: str, src_dir: Path, module_dir: Path) ->
         LOG.info("Using custom path for database module at '%s'", config_module_path)
         return config_module_path
 
+    # Otherwise a source dir must be given.
+    if src_dir is None:
+        raise UsageError("The legacy tokenizer cannot be used with the Nominatim pip module.")
+
     # Compatibility mode for builddir installations.
     if module_dir.exists() and src_dir.samefile(module_dir):
         LOG.info('Running from build directory. Leaving database module as is.')

src/nominatim_db/tools/exec_utils.py

@@ -7,14 +7,17 @@
 """
 Helper functions for executing external programs.
 """
-from typing import Any, Mapping
+from typing import Any, Mapping, List, Optional
 import logging
 import os
+import re
 import subprocess
 import shutil
 
 from ..typing import StrPath
 from ..db.connection import get_pg_env
+from ..errors import UsageError
+from ..version import OSM2PGSQL_REQUIRED_VERSION
 
 LOG = logging.getLogger()
 
@@ -28,9 +31,12 @@ def run_php_server(server_address: str, base_dir: StrPath) -> None:
 def run_osm2pgsql(options: Mapping[str, Any]) -> None:
     """ Run osm2pgsql with the given options.
     """
+    _check_osm2pgsql_version(options['osm2pgsql'])
+
     env = get_pg_env(options['dsn'])
 
     cmd = [_find_osm2pgsql_cmd(options['osm2pgsql']),
+           '--append' if options['append'] else '--create',
            '--slim',
            '--log-progress', 'true',
            '--number-processes', '1' if options['append'] else str(options['threads']),
@@ -42,25 +48,20 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
         env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'),
                                     os.environ.get('LUAPATH', ';')))
         cmd.extend(('--output', 'flex'))
+
+        for flavour in ('data', 'index'):
+            if options['tablespaces'][f"main_{flavour}"]:
+                env[f"NOMINATIM_TABLESPACE_PLACE_{flavour.upper()}"] = \
+                    options['tablespaces'][f"main_{flavour}"]
     else:
         cmd.extend(('--output', 'gazetteer', '--hstore', '--latlon'))
+        cmd.extend(_mk_tablespace_options('main', options))
 
-    cmd.append('--append' if options['append'] else '--create')
 
     if options['flatnode_file']:
         cmd.extend(('--flat-nodes', options['flatnode_file']))
 
-    for key, param in (('slim_data', '--tablespace-slim-data'),
+    cmd.extend(_mk_tablespace_options('slim', options))
-                       ('slim_index', '--tablespace-slim-index'),
-                       ('main_data', '--tablespace-main-data'),
-                       ('main_index', '--tablespace-main-index')):
-        if options['tablespaces'][key]:
-            cmd.extend((param, options['tablespaces'][key]))
-
-    if options['tablespaces']['main_data']:
-        env['NOMINATIM_TABLESPACE_PLACE_DATA'] = options['tablespaces']['main_data']
-    if options['tablespaces']['main_index']:
-        env['NOMINATIM_TABLESPACE_PLACE_INDEX'] = options['tablespaces']['main_index']
 
     if options.get('disable_jit', False):
         env['PGOPTIONS'] = '-c jit=off -c max_parallel_workers_per_gather=0'
@@ -78,12 +79,41 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
                    env=env, check=True)
 
 
-def _find_osm2pgsql_cmd(cmdline: str) -> str:
+def _mk_tablespace_options(ttype: str, options: Mapping[str, Any]) -> List[str]:
+    cmds: List[str] = []
+    for flavour in ('data', 'index'):
+        if options['tablespaces'][f"{ttype}_{flavour}"]:
+            cmds.extend((f"--tablespace-{ttype}-{flavour}",
+                         options['tablespaces'][f"{ttype}_{flavour}"]))
+
+    return cmds
+
+
+def _find_osm2pgsql_cmd(cmdline: Optional[str]) -> str:
     if cmdline is not None:
         return cmdline
 
     in_path = shutil.which('osm2pgsql')
     if in_path is None:
-        raise RuntimeError('osm2pgsql executable not found. Please install osm2pgsql first.')
+        raise UsageError('osm2pgsql executable not found. Please install osm2pgsql first.')
 
     return str(in_path)
+
+
+def _check_osm2pgsql_version(cmdline: Optional[str]) -> None:
+    cmd = [_find_osm2pgsql_cmd(cmdline), '--version']
+
+    result = subprocess.run(cmd, capture_output=True, check=True)
+
+    if not result.stderr:
+        raise UsageError("osm2pgsql does not print version information.")
+
+    verinfo = result.stderr.decode('UTF-8')
+
+    match = re.search(r'osm2pgsql version (\d+)\.(\d+)', verinfo)
+    if match is None:
+        raise UsageError(f"No version information found in output: {verinfo}")
+
+    if (int(match[1]), int(match[2])) < OSM2PGSQL_REQUIRED_VERSION:
+        raise UsageError(f"osm2pgsql is too old. Found version {match[1]}.{match[2]}. "
+                         f"Need at least version {'.'.join(map(str, OSM2PGSQL_REQUIRED_VERSION))}.")

src/nominatim_db/version.py

@@ -62,6 +62,7 @@ NOMINATIM_VERSION = parse_version('4.4.99-1')
 
 POSTGRESQL_REQUIRED_VERSION = (9, 6)
 POSTGIS_REQUIRED_VERSION = (2, 2)
+OSM2PGSQL_REQUIRED_VERSION = (1, 8)
 
 # Cmake sets a variable @GIT_HASH@ by executing 'git --log'. It is not run
 # on every execution of 'make'.

test/python/tools/conftest.py

@@ -7,10 +7,23 @@
 import pytest
 
 @pytest.fixture
-def osm2pgsql_options(temp_db):
+def osm2pgsql_options(temp_db, tmp_path):
-    """ A standard set of options for osm2pgsql.
+    """ A standard set of options for osm2pgsql
+        together with a osm2pgsql mock that just reflects the command line.
     """
-    return dict(osm2pgsql='echo',
+    osm2pgsql_exec = tmp_path / 'osm2pgsql_mock'
+
+    osm2pgsql_exec.write_text("""#!/bin/sh
+
+if [ "$*" = "--version" ]; then
+  >&2 echo "2024-08-09 11:16:23  osm2pgsql version 11.7.2 (11.7.2)"
+else
+  echo "$@"
+fi
+    """)
+    osm2pgsql_exec.chmod(0o777)
+
+    return dict(osm2pgsql=str(osm2pgsql_exec),
                 osm2pgsql_cache=10,
                 osm2pgsql_style='style.file',
                 threads=1,