adapt docs for release

docs/admin/Installation.md

@@ -110,14 +110,17 @@ Then you can install Nominatim with:
 
     pip install nominatim-db nominatim-api
 
-## Downloading and building Nominatim
+## Downloading and building Nominatim from source
+
+The following instructions are only relevant, if you want to build and
+install Nominatim **from source**.
 
 ### Downloading the latest release
 
 You can download the [latest release from nominatim.org](https://nominatim.org/downloads/).
 The release contains all necessary files. Just unpack it.
 
-### Downloading the latest development version
+### Downloading the source for the latest development version
 
 If you want to install latest development version from github:
 
@@ -131,7 +134,7 @@ The development version does not include the country grid. Download it separatel
 wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
 ```
 
-### Building Nominatim
+### Building Nominatim from source
 
 Nominatim is easiest to run from its own virtual environment. To create one, run:
 

docs/develop/Testing.md

@@ -52,15 +52,6 @@ To run the functional tests, do
 
     pytest test/bdd
 
-You can run a single feature file using expression matching:
-
-    pytest test/bdd -k osm2pgsql/import/entrances.feature
-
-This even works for running single tests by adding the line number of the
-scenario header like that:
-
-    pytest test/bdd -k 'osm2pgsql/import/entrances.feature and L4'
-
 The BDD tests create databases for the tests. You can set name of the databases
 through configuration variables in your `pytest.ini`:
 

lib-lua/themes/nominatim/presets.lua

@@ -321,6 +321,7 @@ module.NAME_TAGS = {}
 
 module.NAME_TAGS.core = {main = {'name', 'name:*',
                                  'int_name', 'int_name:*',
+                                 'nat_name', 'nat_name:*',
                                  'reg_name', 'reg_name:*',
                                  'loc_name', 'loc_name:*',
                                  'old_name', 'old_name:*',

lib-sql/functions/placex_triggers.sql

@@ -341,22 +341,6 @@ BEGIN
     END IF;
   END IF;
 
-  IF bnd.extratags ? 'wikidata' THEN
-    FOR linked_placex IN
-      SELECT * FROM placex
-      WHERE placex.class = 'place' AND placex.osm_type = 'N'
-        AND placex.extratags ? 'wikidata' -- needed to select right index
-        AND placex.extratags->'wikidata' = bnd.extratags->'wikidata'
-        AND (placex.linked_place_id is null or placex.linked_place_id = bnd.place_id)
-        AND placex.rank_search < 26
-        AND _st_covers(bnd.geometry, placex.geometry)
-      ORDER BY lower(name->'name') = bnd_name desc
-    LOOP
-      {% if debug %}RAISE WARNING 'Found wikidata-matching place node %', linked_placex.osm_id;{% endif %}
-      RETURN linked_placex;
-    END LOOP;
-  END IF;
-
   -- If extratags has a place tag, look for linked nodes by their place type.
   -- Area and node still have to have the same name.
   IF bnd.extratags ? 'place' and bnd.extratags->'place' != 'postcode'
@@ -377,6 +361,22 @@ BEGIN
     END LOOP;
   END IF;
 
+  IF bnd.extratags ? 'wikidata' THEN
+    FOR linked_placex IN
+      SELECT * FROM placex
+      WHERE placex.class = 'place' AND placex.osm_type = 'N'
+        AND placex.extratags ? 'wikidata' -- needed to select right index
+        AND placex.extratags->'wikidata' = bnd.extratags->'wikidata'
+        AND (placex.linked_place_id is null or placex.linked_place_id = bnd.place_id)
+        AND placex.rank_search < 26
+        AND _st_covers(bnd.geometry, placex.geometry)
+      ORDER BY lower(name->'name') = bnd_name desc
+    LOOP
+      {% if debug %}RAISE WARNING 'Found wikidata-matching place node %', linked_placex.osm_id;{% endif %}
+      RETURN linked_placex;
+    END LOOP;
+  END IF;
+
   -- Name searches can be done for ways as well as relations
   IF bnd_name is not null THEN
     {% if debug %}RAISE WARNING 'Looking for nodes with matching names';{% endif %}
@@ -874,7 +874,7 @@ BEGIN
   -- Remove linkage, if we have computed a different new linkee.
   UPDATE placex SET linked_place_id = null, indexed_status = 2
     WHERE linked_place_id = NEW.place_id
-          and (linked_place is null or place_id != linked_place);
+          and (linked_place is null or linked_place_id != linked_place);
   -- update not necessary for osmline, cause linked_place_id does not exist
 
   -- Postcodes are just here to compute the centroids. They are not searchable

mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Nominatim Manual
+site_name: Nominatim 5.2.0 Manual
 theme:
   font: false
   name: material

src/nominatim_api/result_formatting.py

@@ -10,7 +10,7 @@ Helper classes and functions for formatting results into API responses.
 from typing import Type, TypeVar, Dict, List, Callable, Any, Mapping, Optional, cast
 from collections import defaultdict
 from pathlib import Path
-import importlib.util
+import importlib
 
 from .server.content_types import CONTENT_JSON
 

src/nominatim_api/server/falcon/server.py

@@ -7,8 +7,6 @@
 """
 Server implementation using the falcon webserver framework.
 """
-from __future__ import annotations
-
 from typing import Optional, Mapping, Any, List, cast
 from pathlib import Path
 import asyncio
@@ -163,7 +161,7 @@ class APIMiddleware:
 
     def __init__(self, project_dir: Path, environ: Optional[Mapping[str, str]]) -> None:
         self.api = NominatimAPIAsync(project_dir, environ)
-        self.app: Optional[App[Request, Response]] = None
+        self.app: Optional[App] = None
 
     @property
     def config(self) -> Configuration:
@@ -171,7 +169,7 @@ class APIMiddleware:
         """
         return self.api.config
 
-    def set_app(self, app: App[Request, Response]) -> None:
+    def set_app(self, app: App) -> None:
         """ Set the Falcon application this middleware is connected to.
         """
         self.app = app
@@ -195,7 +193,7 @@ class APIMiddleware:
 
 
 def get_application(project_dir: Path,
-                    environ: Optional[Mapping[str, str]] = None) -> App[Request, Response]:
+                    environ: Optional[Mapping[str, str]] = None) -> App:
     """ Create a Nominatim Falcon ASGI application.
     """
     apimw = APIMiddleware(project_dir, environ)
@@ -217,7 +215,7 @@ def get_application(project_dir: Path,
     return app
 
 
-def run_wsgi() -> App[Request, Response]:
+def run_wsgi() -> App:
     """ Entry point for uvicorn.
 
         Make sure uvicorn is run from the project directory.

src/nominatim_db/clicmd/setup.py

@@ -23,7 +23,6 @@ from ..tokenizer.base import AbstractTokenizer
 from ..version import NOMINATIM_VERSION
 from .args import NominatimArgs
 
-import time
 
 LOG = logging.getLogger()
 
@@ -87,8 +86,6 @@ class SetupAll:
         from ..tools import database_import, postcodes, freeze
         from ..indexer.indexer import Indexer
 
-        start_time = time.time()
-
         num_threads = args.threads or psutil.cpu_count() or 1
         country_info.setup_country_config(args.config)
 
@@ -141,10 +138,6 @@ class SetupAll:
         LOG.warning('Recompute word counts')
         tokenizer.update_statistics(args.config, threads=num_threads)
 
-        end_time = time.time()
-        elapsed = end_time - start_time
-        LOG.warning(f'Import completed successfully in {elapsed:.2f} seconds.')
-
         self._finalize_database(args.config.get_libpq_dsn(), args.offline)
 
         return 0

src/nominatim_db/data/postcode_format.py

@@ -29,9 +29,6 @@ class CountryPostcodeMatcher:
         self.norm_pattern = re.compile(f'\\s*(?:{country_code.upper()}[ -]?)?({pc_pattern})\\s*')
         self.pattern = re.compile(pc_pattern)
 
-        # We want to exclude 0000, 00-000, 000 00 etc
-        self.zero_pattern = re.compile(r'^[0\- ]+$')
-
         self.output = config.get('output', r'\g<0>')
 
     def match(self, postcode: str) -> Optional[Match[str]]:
@@ -43,10 +40,7 @@ class CountryPostcodeMatcher:
         normalized = self.norm_pattern.fullmatch(postcode.upper())
 
         if normalized:
-            match = self.pattern.fullmatch(normalized.group(1))
-            if match and self.zero_pattern.match(match.string):
-                return None
-            return match
+            return self.pattern.fullmatch(normalized.group(1))
 
         return None
 

test/bdd/conftest.py

@@ -9,7 +9,6 @@ Fixtures for BDD test steps
 """
 import sys
 import json
-import re
 from pathlib import Path
 
 import psycopg
@@ -21,8 +20,7 @@ sys.path.insert(0, str(SRC_DIR / 'src'))
 
 import pytest
 from pytest_bdd.parsers import re as step_parse
-from pytest_bdd import given, when, then, scenario
-from pytest_bdd.feature import get_features
+from pytest_bdd import given, when, then
 
 pytest.register_assert_rewrite('utils')
 
@@ -375,57 +373,3 @@ def check_place_missing_lines(db_conn, table, osm_type, osm_id, osm_class):
 
     with db_conn.cursor() as cur:
         assert cur.execute(sql, params).fetchone()[0] == 0
-
-
-if pytest.version_tuple >= (8, 0, 0):
-    def pytest_pycollect_makemodule(module_path, parent):
-        return BddTestCollector.from_parent(parent, path=module_path)
-
-
-class BddTestCollector(pytest.Module):
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def collect(self):
-        for item in super().collect():
-            yield item
-
-        if hasattr(self.obj, 'PYTEST_BDD_SCENARIOS'):
-            for path in self.obj.PYTEST_BDD_SCENARIOS:
-                for feature in get_features([str(Path(self.path.parent, path).resolve())]):
-                    yield FeatureFile.from_parent(self,
-                                                  name=str(Path(path, feature.rel_filename)),
-                                                  path=Path(feature.filename),
-                                                  feature=feature)
-
-
-# borrowed from pytest-bdd: src/pytest_bdd/scenario.py
-def make_python_name(string: str) -> str:
-    """Make python attribute name out of a given string."""
-    string = re.sub(r"\W", "", string.replace(" ", "_"))
-    return re.sub(r"^\d+_*", "", string).lower()
-
-
-class FeatureFile(pytest.File):
-    class obj:
-        pass
-
-    def __init__(self, feature, **kwargs):
-        self.feature = feature
-        super().__init__(**kwargs)
-
-    def collect(self):
-        for sname, sobject in self.feature.scenarios.items():
-            class_name = f"L{sobject.line_number}"
-            test_name = "test_" + make_python_name(sname)
-
-            @scenario(self.feature.filename, sname)
-            def _test():
-                pass
-
-            tclass = type(class_name, (),
-                          {test_name: staticmethod(_test)})
-            setattr(self.obj, class_name, tclass)
-
-            yield pytest.Class.from_parent(self, name=class_name, obj=tclass)

test/bdd/test_api.py

@@ -15,7 +15,7 @@ import xml.etree.ElementTree as ET
 
 import pytest
 from pytest_bdd.parsers import re as step_parse
-from pytest_bdd import when, given, then
+from pytest_bdd import scenarios, when, given, then
 
 from nominatim_db import cli
 from nominatim_db.config import Configuration
@@ -150,8 +150,4 @@ def parse_api_json_response(api_response, fmt, num):
     return result
 
 
-if pytest.version_tuple >= (8, 0, 0):
-    PYTEST_BDD_SCENARIOS = ['features/api']
-else:
-    from pytest_bdd import scenarios
-    scenarios('features/api')
+scenarios('features/api')

test/bdd/test_db.py

@@ -15,7 +15,7 @@ import re
 import psycopg
 
 import pytest
-from pytest_bdd import when, then, given
+from pytest_bdd import scenarios, when, then, given
 from pytest_bdd.parsers import re as step_parse
 
 from utils.place_inserter import PlaceColumn
@@ -276,8 +276,4 @@ def then_check_interpolation_table_negative(db_conn, oid):
         assert cur.fetchone()[0] == 0
 
 
-if pytest.version_tuple >= (8, 0, 0):
-    PYTEST_BDD_SCENARIOS = ['features/db']
-else:
-    from pytest_bdd import scenarios
-    scenarios('features/db')
+scenarios('features/db')

test/bdd/test_osm2pgsql.py

@@ -11,7 +11,7 @@ import asyncio
 import random
 
 import pytest
-from pytest_bdd import when, then, given
+from pytest_bdd import scenarios, when, then, given
 from pytest_bdd.parsers import re as step_parse
 
 from nominatim_db import cli
@@ -106,8 +106,4 @@ def check_place_content(db_conn, datatable, node_grid, table, exact):
     check_table_content(db_conn, table, datatable, grid=node_grid, exact=bool(exact))
 
 
-if pytest.version_tuple >= (8, 0, 0):
-    PYTEST_BDD_SCENARIOS = ['features/osm2pgsql']
-else:
-    from pytest_bdd import scenarios
-    scenarios('features/osm2pgsql')
+scenarios('features/osm2pgsql')

test/python/tokenizer/sanitizers/test_clean_postcodes.py

@@ -237,9 +237,3 @@ def test_postcode_default_pattern_pass(sanitize, postcode):
 @pytest.mark.sanitizer_params(convert_to_address=False, default_pattern='[A-Z0-9- ]{3,12}')
 def test_postcode_default_pattern_fail(sanitize, postcode):
     assert sanitize(country='an', postcode=postcode) == []
-
-
-@pytest.mark.parametrize("postcode", ('00000', '00-000', 'PL-00000', 'PL 00-000'))
-@pytest.mark.sanitizer_params(convert_to_address=False)
-def test_postcode_zeros(sanitize, postcode):
-    assert sanitize(country='pl', postcode=postcode) == []

vagrant/Install-on-Ubuntu-22.sh

@@ -25,7 +25,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         libbz2-dev libpq-dev liblua5.3-dev lua5.3 lua-dkjson \
                         nlohmann-json3-dev postgresql-14-postgis-3 \
                         postgresql-contrib-14 postgresql-14-postgis-3-scripts \
-                        libicu-dev virtualenv git
+                        libicu-dev virtualenv
 
 #
 # System Configuration
@@ -97,23 +97,6 @@ fi                                    #DOCS:
 # Building and Configuration
 # --------------------------
 #
-# Get the source code from Github and change into the source directory
-#
-if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
-    cd $USERHOME
-    git clone https://github.com/osm-search/Nominatim.git
-    cd Nominatim
-else                               #DOCS:
-    cd $USERHOME/Nominatim         #DOCS:
-fi                                 #DOCS:
-
-# When installing the latest source from github, you also need to
-# download the country grid:
-
-if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
-    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
-fi                                 #DOCS:
-
 # Nominatim needs osm2pgsql >= 1.8. The version that comes with Ubuntu is
 # too old. Download and compile your own:
 
@@ -124,7 +107,6 @@ fi                                 #DOCS:
     cmake ../osm2pgsql
     make
     sudo make install
-    cd $USERHOME/Nominatim
 
 # Nominatim should be installed in a separate Python virtual environment.
 # Create the virtual environment:
@@ -137,8 +119,7 @@ fi                                 #DOCS:
 
 # Now install Nominatim using pip:
 
-    cd $USERHOME/Nominatim
-    $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-db
+    $USERHOME/nominatim-venv/bin/pip install nominatim-db
 
 # Nominatim is now ready to use. You can continue with
 # [importing a database from OSM data](../admin/Import.md). If you want to set up
@@ -154,9 +135,7 @@ fi                                 #DOCS:
 # To install all packages, run:
 
 #DOCS:```sh
-$USERHOME/nominatim-venv/bin/pip install falcon uvicorn gunicorn
-cd $USERHOME/Nominatim
-$USERHOME/nominatim-venv/bin/pip install packaging/nominatim-api
+$USERHOME/nominatim-venv/bin/pip install falcon uvicorn gunicorn nominatim-api
 #DOCS:```
 
 

vagrant/Install-on-Ubuntu-24.sh

@@ -21,7 +21,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
 # Now you can install all packages needed for Nominatim:
 
     sudo apt-get install -y osm2pgsql postgresql-postgis postgresql-postgis-scripts \
-                            pkg-config libicu-dev virtualenv git
+                            pkg-config libicu-dev virtualenv
 
 
 #
@@ -94,23 +94,6 @@ fi                                    #DOCS:
 # Building and Configuration
 # --------------------------
 #
-# Get the source code from Github and change into the source directory
-#
-if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
-    cd $USERHOME
-    git clone https://github.com/osm-search/Nominatim.git
-    cd Nominatim
-else                               #DOCS:
-    cd $USERHOME/Nominatim         #DOCS:
-fi                                 #DOCS:
-
-# When installing the latest source from github, you also need to
-# download the country grid:
-
-if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
-    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
-fi                                 #DOCS:
-
 # Nominatim should be installed in a separate Python virtual environment.
 # Create the virtual environment:
 
@@ -122,8 +105,7 @@ fi                                 #DOCS:
 
 # Now install Nominatim using pip:
 
-    cd $USERHOME/Nominatim
-    $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-db
+    $USERHOME/nominatim-venv/bin/pip install nominatim-db
 
 # Nominatim is now ready to use. The nominatim binary is available at
 # `$USERHOME/venv/bin/nominatim`. If you want to have 'nominatim' in your
@@ -147,9 +129,7 @@ fi                                 #DOCS:
 # To install all packages, run:
 
 #DOCS:```sh
-$USERHOME/nominatim-venv/bin/pip install falcon uvicorn gunicorn
-cd $USERHOME/Nominatim
-$USERHOME/nominatim-venv/bin/pip install packaging/nominatim-api
+$USERHOME/nominatim-venv/bin/pip install falcon uvicorn gunicorn nominatim-api
 #DOCS:```
 
 # Next you need to create a systemd job that runs Nominatim on gunicorn.