prepare release 4.4.1

The psycopg dialect was only added in SQLAlchemy 2.0. To avoid loading
errors when SQLAlchemy 1.4 is installed together with psycopg3,
check that the dialect is really available.

.github/workflows/ci-tests.yml

@@ -65,7 +65,7 @@ jobs:
             - name: Unpack Nominatim
               run: tar xf nominatim-src.tar.bz2
 
-            - uses: actions/setup-python@v4
+            - uses: actions/setup-python@v5
               with:
                 python-version: 3.7
               if: matrix.flavour == 'oldstuff'
@@ -80,23 +80,23 @@ jobs:
                   flavour: ${{ matrix.flavour }}
                   lua: ${{ matrix.lua }}
 
-            - name: Install test prerequisites (behave from apt)
+            - name: Install test prerequsites (behave from apt)
               run: sudo apt-get install -y -qq python3-behave
               if: matrix.flavour == 'ubuntu-20'
 
-            - name: Install test prerequisites (behave from pip)
+            - name: Install test prerequsites (behave from pip)
               run: pip3 install behave==1.2.6
               if: (matrix.flavour == 'oldstuff') || (matrix.flavour == 'ubuntu-22')
 
-            - name: Install test prerequisites (from apt for Ununtu 2x)
+            - name: Install test prerequsites (from apt for Ununtu 2x)
               run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn
               if: matrix.flavour != 'oldstuff'
 
             - name: Install newer pytest-asyncio
-              run: pip3 install -U pytest-asyncio==0.21.1
+              run: pip3 install -U pytest-asyncio
               if: matrix.flavour == 'ubuntu-20'
 
-            - name: Install test prerequisites (from pip for Ubuntu 18)
+            - name: Install test prerequsites (from pip for Ubuntu 18)
               run: pip3 install pytest pytest-asyncio uvicorn
               if: matrix.flavour == 'oldstuff'
 
@@ -105,12 +105,12 @@ jobs:
 
             - name: Install latest pylint
               run: pip3 install -U pylint
-              if: matrix.flavour != 'oldstuff'
+              if: matrix.flavour == 'ubuntu-22'
 
             - name: Python linting
               run: python3 -m pylint nominatim
               working-directory: Nominatim
-              if: matrix.flavour != 'oldstuff'
+              if: matrix.flavour == 'ubuntu-22'
 
             - name: Python unit tests
               run: python3 -m pytest test/python
@@ -165,7 +165,7 @@ jobs:
               with:
                   cmake-args: -DBUILD_MODULE=on
 
-            - name: Install test prerequisites
+            - name: Install test prerequsites
               run: sudo apt-get install -y -qq python3-behave
 
             - name: BDD tests (legacy tokenizer)
@@ -212,7 +212,7 @@ jobs:
               with:
                   flavour: 'ubuntu-22'
 
-            - name: Install test prerequisites
+            - name: Install test prerequsites
               run: sudo apt-get install -y -qq python3-behave
 
             - name: BDD tests (php)

CMakeLists.txt

@@ -20,7 +20,7 @@ project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 4)
 set(NOMINATIM_VERSION_MINOR 4)
-set(NOMINATIM_VERSION_PATCH 0)
+set(NOMINATIM_VERSION_PATCH 1)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 

ChangeLog

@@ -1,3 +1,10 @@
+4.4.1
+ * fix geocodejson output: admin level output should only print boundaries
+ * updating: restrict invalidation of child objects on large street features
+ * restrict valid interpolation house numbers to 0-999999
+ * fix import error when SQLAlchemy 1.4 and psycopg3 are installed
+ * various typo fixes in the documentation
+
 4.4.0
  * add export to SQLite database and SQLite support for the frontend
  * switch to Python frontend as the default frontend

docs/api/Lookup.md

@@ -93,7 +93,7 @@ as the ["Accept-Language" HTTP header](https://developer.mozilla.org/en-US/docs/
     results when using Nominatim in the browser versus in a command-line tool
     like wget or curl. The command-line tools
     usually don't send any Accept-Language header, prompting Nominatim
-    to show results in the local language. Browsers on the contrary always
+    to show results in the local language. Browsers on the contratry always
     send the currently chosen browser language.
 
 

docs/api/Reverse.md

@@ -115,7 +115,7 @@ as the ["Accept-Language" HTTP header](https://developer.mozilla.org/en-US/docs/
     results when using Nominatim in the browser versus in a command-line tool
     like wget or curl. The command-line tools
     usually don't send any Accept-Language header, prompting Nominatim
-    to show results in the local language. Browsers on the contrary always
+    to show results in the local language. Browsers on the contratry always
     send the currently chosen browser language.
 
 

docs/api/Search.md

@@ -152,7 +152,7 @@ as the ["Accept-Language" HTTP header](https://developer.mozilla.org/en-US/docs/
     results when using Nominatim in the browser versus in a command-line tool
     like wget or curl. The command-line tools
     usually don't send any Accept-Language header, prompting Nominatim
-    to show results in the local language. Browsers on the contrary always
+    to show results in the local language. Browsers on the contratry always
     send the currently chosen browser language.
 
 ### Result restriction
@@ -165,7 +165,7 @@ results and thus give a preference to some results over others.
 |-----------| ----- | ------- |
 | countrycodes | comma-separated list of country codes | _unset_ |
 
-Filer that limits the search results to one or more countries.
+Filter that limits the search results to one or more countries.
 The country code must be the
 [ISO 3166-1alpha2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) code
 of the country, e.g. `gb` for the United Kingdom, `de` for Germany.

docs/customize/Tokenizers.md

@@ -52,7 +52,7 @@ NOMINATIM_DATABASE_MODULE_PATH=<path to directory where nominatim.so resides>
 ```
 
 This is in particular useful when the database runs on a different server.
-See [Advanced installations](../admin/Advanced-Installations.md#using-an-external-postgresql-database) for details.
+See [Advanced installations](../admin/Advanced-Installations.md#importing-nominatim-to-an-external-postgresql-database) for details.
 
 There are no other configuration options for the legacy tokenizer. All
 normalization functions are hard-coded.

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Nominatim 4.4.1 Manual
+site_name: Nominatim Manual
 theme:
   name: material
   features:
@@ -68,7 +68,7 @@ markdown_extensions:
         alternate_style: true
     - def_list
     - toc:
-        permalink: 🔗
+        permalink: 
 extra_css: [extra.css, styles.css]
 plugins:
     - search

lib-sql/functions/interpolation.sql

@@ -219,10 +219,11 @@ BEGIN
           -- formatted postcode and therefore 'postcode' contains a derived
           -- variant.
           CASE WHEN address ? 'postcode' THEN placex.postcode ELSE NULL::text END as postcode,
-          substring(address->'housenumber','[0-9]+')::integer as hnr
+          (address->'housenumber')::integer as hnr
         FROM placex, generate_series(1, array_upper(waynodes, 1)) nodeidpos
         WHERE osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT
               and address is not NULL and address ? 'housenumber'
+              and address->'housenumber' ~ '^[0-9]{1,6}$'
               and ST_Distance(NEW.linegeo, geometry) < 0.0005
         ORDER BY nodeidpos
     LOOP

lib-sql/functions/placex_triggers.sql

@@ -731,10 +731,12 @@ BEGIN
 {% if not disable_diff_updates %}
   -- The following is not needed until doing diff updates, and slows the main index process down
 
-  IF NEW.rank_address > 0 THEN
+  IF NEW.rank_address between 2 and 27 THEN
     IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
       -- Performance: We just can't handle re-indexing for country level changes
-      IF st_area(NEW.geometry) < 1 THEN
+      IF (NEW.rank_address < 26 and st_area(NEW.geometry) < 1)
+         OR (NEW.rank_address >= 26 and st_area(NEW.geometry) < 0.01)
+      THEN
         -- mark items within the geometry for re-indexing
   --    RAISE WARNING 'placex poly insert: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
 
@@ -749,9 +751,11 @@ BEGIN
                     or name is not null
                     or (NEW.rank_address >= 16 and address ? 'place'));
       END IF;
-    ELSE
+    ELSEIF ST_GeometryType(NEW.geometry) not in ('ST_LineString', 'ST_MultiLineString')
+           OR ST_Length(NEW.geometry) < 0.5
+    THEN
       -- mark nearby items for re-indexing, where 'nearby' depends on the features rank_search and is a complete guess :(
-      diameter := update_place_diameter(NEW.rank_search);
+      diameter := update_place_diameter(NEW.rank_address);
       IF diameter > 0 THEN
   --      RAISE WARNING 'placex point insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,diameter;
         IF NEW.rank_search >= 26 THEN

nominatim/api/v1/format_json.py

@@ -247,7 +247,8 @@ def format_base_geocodejson(results: Union[napi.ReverseResults, napi.SearchResul
             out.key('admin').start_object()
             if result.address_rows:
                 for line in result.address_rows:
-                    if line.isaddress and (line.admin_level or 15) < 15 and line.local_name:
+                    if line.isaddress and (line.admin_level or 15) < 15 and line.local_name \
+                       and line.category[0] == 'boundary' and line.category[1] == 'administrative':
                         out.keyval(f"level{line.admin_level}", line.local_name)
             out.end_object().next()
 

nominatim/clicmd/replication.py

@@ -142,7 +142,7 @@ class UpdateReplication:
             if not args.do_index:
                 LOG.fatal("Indexing cannot be disabled when running updates continuously.")
                 raise UsageError("Bad argument '--no-index'.")
-            recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
+        recheck_interval = args.config.get_int('REPLICATION_RECHECK_INTERVAL')
 
         tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
         indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, args.threads or 1)

nominatim/db/async_core_library.py

@@ -7,15 +7,17 @@
 """
 Import the base library to use with asynchronous SQLAlchemy.
 """
-# pylint: disable=invalid-name
+# pylint: disable=invalid-name, ungrouped-imports, unused-import
 
 from typing import Any
 
 try:
+    import sqlalchemy.dialects.postgresql.psycopg
     import psycopg
     PGCORE_LIB = 'psycopg'
     PGCORE_ERROR: Any = psycopg.Error
 except ModuleNotFoundError:
+    import sqlalchemy.dialects.postgresql.asyncpg
     import asyncpg
     PGCORE_LIB = 'asyncpg'
     PGCORE_ERROR = asyncpg.PostgresError

nominatim/version.py

@@ -34,7 +34,7 @@ class NominatimVersion(NamedTuple):
         return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}"
 
 
-NOMINATIM_VERSION = NominatimVersion(4, 4, 0, 0)
+NOMINATIM_VERSION = NominatimVersion(4, 4, 1, 0)
 
 POSTGRESQL_REQUIRED_VERSION = (9, 6)
 POSTGIS_REQUIRED_VERSION = (2, 2)

vagrant/Install-on-Ubuntu-20.sh

@@ -27,7 +27,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-contrib-12 postgresql-12-postgis-3-scripts \
                         libicu-dev python3-dotenv \
                         python3-psycopg2 python3-psutil python3-jinja2 python3-pip \
-                        python3-icu python3-datrie python3-yaml
+                        python3-icu python3-datrie python3-yaml git
 
 # Some of the Python packages that come with Ubuntu 20.04 are too old, so
 # install the latest version from pip:
@@ -109,18 +109,25 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.4.1.tar.bz2
+    git clone --recursive https://github.com/openstreetmap/Nominatim.git
-    tar xf Nominatim-4.4.1.tar.bz2
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.4.1
+    cmake $USERHOME/Nominatim
     make
     sudo make install
 

vagrant/Install-on-Ubuntu-22.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         libicu-dev python3-dotenv \
                         python3-psycopg2 python3-psutil python3-jinja2 \
                         python3-sqlalchemy python3-asyncpg \
-                        python3-icu python3-datrie python3-yaml
+                        python3-icu python3-datrie python3-yaml git
 
 
 #
@@ -105,8 +105,8 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.4.1.tar.bz2
+    git clone --recursive https://github.com/openstreetmap/Nominatim.git
-    tar xf Nominatim-4.4.1.tar.bz2
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
@@ -123,7 +123,7 @@ fi                                 #DOCS:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.4.1
+    cmake $USERHOME/Nominatim
     make
     sudo make install