adapt to release 4.3.4

* fix typo in docs (NOMINATIM_LOG_FILE uses s not ms)

.github/actions/setup-postgresql/action.yml

@@ -15,9 +15,7 @@ runs:
         - name: Remove existing PostgreSQL
           run: |
               sudo apt-get purge -yq postgresql*
-              sudo apt install curl ca-certificates gnupg
-              curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
-              sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+              sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
               sudo apt-get update -qq
 
           shell: bash

.github/workflows/ci-tests.yml

@@ -37,8 +37,13 @@ jobs:
         needs: create-archive
         strategy:
             matrix:
-                ubuntu: [20, 22]
+                ubuntu: [18, 20, 22]
                 include:
+                    - ubuntu: 18
+                      postgresql: 9.6
+                      postgis: 2.5
+                      pytest: pytest
+                      php: 7.2
                     - ubuntu: 20
                       postgresql: 13
                       postgis: 3
@@ -64,10 +69,8 @@ jobs:
               uses: shivammathur/setup-php@v2
               with:
                   php-version: ${{ matrix.php }}
-                  tools: phpunit:9, phpcs, composer
+                  tools: phpunit, phpcs, composer
                   ini-values: opcache.jit=disable
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
             - uses: actions/setup-python@v4
               with:
@@ -97,22 +100,18 @@ jobs:
 
             - name: Install latest pylint/mypy
               run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions
-              if: matrix.ubuntu == 22
 
             - name: PHP linting
               run: phpcs --report-width=120 .
               working-directory: Nominatim
-              if: matrix.ubuntu == 22
 
             - name: Python linting
               run: pylint nominatim
               working-directory: Nominatim
-              if: matrix.ubuntu == 22
 
             - name: Python static typechecking
               run: mypy --strict nominatim
               working-directory: Nominatim
-              if: matrix.ubuntu == 22
 
 
             - name: PHP unit tests

.pylintrc

@@ -13,6 +13,6 @@ ignored-classes=NominatimArgs,closing
 # 'too-many-ancestors' is triggered already by deriving from UserDict
 # 'not-context-manager' disabled because it causes false positives once
 #   typed Python is enabled. See also https://github.com/PyCQA/pylint/issues/5273
-disable=too-few-public-methods,duplicate-code,too-many-ancestors,bad-option-value,no-self-use,not-context-manager,use-dict-literal
+disable=too-few-public-methods,duplicate-code,too-many-ancestors,bad-option-value,no-self-use,not-context-manager
 
 good-names=i,x,y,m,fd,db,cc

CMakeLists.txt

@@ -20,7 +20,7 @@ project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 4)
 set(NOMINATIM_VERSION_MINOR 2)
-set(NOMINATIM_VERSION_PATCH 4)
+set(NOMINATIM_VERSION_PATCH 0)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 
@@ -269,12 +269,6 @@ install(FILES settings/env.defaults
               settings/import-address.style
               settings/import-full.style
               settings/import-extratags.style
-              settings/import-admin.lua
-              settings/import-street.lua
-              settings/import-address.lua
-              settings/import-full.lua
-              settings/import-extratags.lua
-              settings/flex-base.lua
               settings/icu_tokenizer.yaml
               settings/country_settings.yaml
         DESTINATION ${NOMINATIM_CONFIGDIR})

ChangeLog

@@ -1,33 +1,3 @@
-4.2.4
- * fix a potential SQL injection in 'nominatim admin --collect-os-info'
- * fix compatibility issue with PostGIS 3.4
-
-4.2.3
-
- * fix deletion handling for 'nominatim add-data'
- * adapt place_force_delete() to new deletion handling
- * flex style: avoid dropping of postcode areas
- * fix update errors on address interpolation handling
-
-4.2.2
-
- * extend flex-style library to fully support all default styles
- * fix handling of Hebrew aleph
- * do not assign postcodes to rivers
- * fix string matching in PHP code
- * update osm2pgsql (various updates to flex)
- * fix slow query when deleting places on update
- * fix CLI details query
- * fix recalculation of importance values
- * fix polygon simplification in reverse results
- * add class/type information to reverse geocodejson result
- * minor improvements to default tokenizer configuration
- * various smaller fixes to documentation
-
-4.2.1
-
- * fix XSS vulnerability in debug view
-
 4.2.0
 
  * add experimental support for osm2pgsql flex style
@@ -51,10 +21,6 @@
  * typing fixes to work with latest type annotations from typeshed
  * smaller improvements to documentation (thanks to @mausch)
 
-4.1.1
-
- * fix XSS vulnerability in debug view
-
 4.1.0
 
  * switch to ICU tokenizer as default
@@ -91,10 +57,6 @@
  * add setup instructions for updates and systemd
  * drop support for PostgreSQL 9.5
 
-4.0.2
-
- * fix XSS vulnerability in debug view
-
 4.0.1
 
  * fix initialisation error in replication script
@@ -133,10 +95,6 @@
  * add testing of installation scripts via CI
  * drop support for Python < 3.6 and Postgresql < 9.5
 
-3.7.3
-
- * fix XSS vulnerability in debug view
-
 3.7.2
 
  * fix database check for reverse-only imports

VAGRANT.md

@@ -1,6 +1,6 @@
 # Install Nominatim in a virtual machine for development and testing
 
-This document describes how you can install Nominatim inside a Ubuntu 22
+This document describes how you can install Nominatim inside a Ubuntu 16
 virtual machine on your desktop/laptop (host machine). The goal is to give
 you a development environment to easily edit code and run the test suite
 without affecting the rest of your system. 
@@ -69,7 +69,8 @@ installation.
 PHP errors are written to `/var/log/apache2/error.log`.
 
 With `echo` and `var_dump()` you write into the output (HTML/XML/JSON) when
-you either add `&debug=1` to the URL.
+you either add `&debug=1` to the URL (preferred) or set
+`@define('CONST_Debug', true);` in `settings/local.php`.
 
 In the Python BDD test you can use `logger.info()` for temporary debug
 statements.
@@ -129,10 +130,6 @@ and then
 Yes, Vagrant and Virtualbox can be installed on MS Windows just fine. You need a 64bit
 version of Windows.
 
-##### Will it run on Apple Silicon?
-
-You might need to replace Virtualbox with [Parallels](https://www.parallels.com/products/desktop/).
-There is no free/open source version of Parallels.
 
 ##### Why Monaco, can I use another country?
 
@@ -144,12 +141,11 @@ No. Long running Nominatim installations will differ once new import features (o
 bug fixes) get added since those usually only get applied to new/changed data.
 
 Also this document skips the optional Wikipedia data import which affects ranking
-of search results. See [Nominatim installation](https://nominatim.org/release-docs/latest/admin/Installation)
-for details.
+of search results. See [Nominatim installation](https://nominatim.org/release-docs/latest/admin/Installation) for details.
 
 ##### Why Ubuntu? Can I test CentOS/Fedora/CoreOS/FreeBSD?
 
-There used to be a Vagrant script for CentOS available, but the Nominatim directory
+There is a Vagrant script for CentOS available, but the Nominatim directory
 isn't symlinked/mounted to the host which makes development trickier. We used
 it mainly for debugging installation with SELinux.
 
@@ -158,17 +154,14 @@ are slightly different, e.g. the name of the package manager, Apache2 package
 name, location of files. We chose Ubuntu because that is closest to the
 nominatim.openstreetmap.org production environment.
 
-You can configure/download other Vagrant boxes from
-[https://app.vagrantup.com/boxes/search](https://app.vagrantup.com/boxes/search).
+You can configure/download other Vagrant boxes from [https://app.vagrantup.com/boxes/search](https://app.vagrantup.com/boxes/search).
 
 ##### How can I connect to an existing database?
 
-Let's say you have a Postgres database named `nominatim_it` on server `your-server.com`
-and port `5432`. The Postgres username is `postgres`. You can edit the `.env` in your
-project directory and point Nominatim to it.
-
-    NOMINATIM_DATABASE_DSN="pgsql:host=your-server.com;port=5432;user=postgres;dbname=nominatim_it
+Let's say you have a Postgres database named `nominatim_it` on server `your-server.com` and port `5432`. The Postgres username is `postgres`. You can edit `settings/local.php` and point Nominatim to it.
 
+    pgsql:host=your-server.com;port=5432;user=postgres;dbname=nominatim_it
+    
 No data import or restarting necessary.
 
 If the Postgres installation is behind a firewall, you can try
@@ -176,12 +169,11 @@ If the Postgres installation is behind a firewall, you can try
     ssh -L 9999:localhost:5432 your-username@your-server.com
 
 inside the virtual machine. It will map the port to `localhost:9999` and then
-you edit `.env` file with
+you edit `settings/local.php` with
 
-    NOMINATIM_DATABASE_DSN="pgsql:host=localhost;port=9999;user=postgres;dbname=nominatim_it"
+    @define('CONST_Database_DSN', 'pgsql:host=localhost;port=9999;user=postgres;dbname=nominatim_it');
 
-To access postgres directly remember to specify the hostname,
-e.g. `psql --host localhost --port 9999 nominatim_it`
+To access postgres directly remember to specify the hostname, e.g. `psql --host localhost --port 9999 nominatim_it`
 
 
 ##### My computer is slow and the import takes too long. Can I start the virtual machine "in the cloud"?

docs/admin/Migration.md

@@ -15,6 +15,14 @@ breaking changes. **Please read them before running the migration.**
     If you are migrating from a version <3.6, then you still have to follow
     the manual migration steps up to 3.6.
 
+## 4.2.2 -> 4.2.3
+
+### Update interpolation functions
+
+When updating to this release, you need to run `nominatim refresh --functions`
+after updating and before restarting updates. Otherwise you may see an error
+`Splitting of Point geometries is unsupported` or similar.
+
 ## 4.0.0 -> 4.1.0
 
 ### ICU tokenizer is the new default

docs/extra.css

@@ -1,7 +1,3 @@
-.toctree-l3 {
-    display: none!important
-}
-
 table {
     margin-bottom: 12pt
 }

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Nominatim Documentation
+site_name: Nominatim 4.2.4
 theme: readthedocs
 docs_dir: ${CMAKE_CURRENT_BINARY_DIR}
 site_url: https://nominatim.org

lib-php/DebugHtml.php

@@ -135,7 +135,7 @@ class Debug
 
     public static function printSQL($sSQL)
     {
-        echo '<p><tt><font color="#aaa">'.htmlspecialchars($sSQL, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401).'</font></tt></p>'."\n";
+        echo '<p><tt><font color="#aaa">'.$sSQL.'</font></tt></p>'."\n";
     }
 
     private static function outputVar($mVar, $sPreNL)
@@ -178,12 +178,11 @@ class Debug
         }
 
         if (is_string($mVar)) {
-            $sOut = "'$mVar'";
-        } else {
-            $sOut = (string)$mVar;
+            echo "'$mVar'";
+            return strlen($mVar) + 2;
         }
 
-        echo htmlspecialchars($sOut, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401);
-        return strlen($sOut);
+        echo (string)$mVar;
+        return strlen((string)$mVar);
     }
 }

lib-php/Geocode.php

@@ -874,7 +874,7 @@ class Geocode
                 $iCountWords = 0;
                 $sAddress = $aResult['langaddress'];
                 foreach ($aRecheckWords as $i => $sWord) {
-                    if (grapheme_stripos($sAddress, $sWord)!==false) {
+                    if (stripos($sAddress, $sWord)!==false) {
                         $iCountWords++;
                         if (preg_match('/(^|,)\s*'.preg_quote($sWord, '/').'\s*(,|$)/', $sAddress)) {
                             $iCountWords += 0.1;

lib-php/PlaceLookup.php

@@ -524,7 +524,12 @@ class PlaceLookup
 
         // Get the bounding box and outline polygon
         $sSQL = 'select place_id,0 as numfeatures,st_area(geometry) as area,';
-        $sSQL .= ' ST_Y(centroid) as centrelat, ST_X(centroid) as centrelon,';
+        if ($fLonReverse != null && $fLatReverse != null) {
+            $sSQL .= ' ST_Y(closest_point) as centrelat,';
+            $sSQL .= ' ST_X(closest_point) as centrelon,';
+        } else {
+            $sSQL .= ' ST_Y(centroid) as centrelat, ST_X(centroid) as centrelon,';
+        }
         $sSQL .= ' ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,';
         $sSQL .= ' ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon';
         if ($this->bIncludePolygonAsGeoJSON) {
@@ -539,21 +544,19 @@ class PlaceLookup
         if ($this->bIncludePolygonAsText) {
             $sSQL .= ',ST_AsText(geometry) as astext';
         }
-
-        $sSQL .= ' FROM (SELECT place_id';
         if ($fLonReverse != null && $fLatReverse != null) {
-            $sSQL .= ',CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
-            $sSQL .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
-            $sSQL .=' ELSE centroid END AS centroid';
+            $sFrom = ' from (SELECT * , CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
+            $sFrom .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
+            $sFrom .=' ELSE centroid END AS closest_point';
+            $sFrom .= ' from placex where place_id = '.$iPlaceID.') as plx';
         } else {
-            $sSQL .= ',centroid';
+            $sFrom = ' from placex where place_id = '.$iPlaceID;
         }
         if ($this->fPolygonSimplificationThreshold > 0) {
-            $sSQL .= ',ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry';
+            $sSQL .= ' from (select place_id,centroid,ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry'.$sFrom.') as plx';
         } else {
-            $sSQL .= ',geometry';
+            $sSQL .= $sFrom;
         }
-        $sSQL .= ' FROM placex where place_id = '.$iPlaceID.') as plx';
 
         $aPointPolygon = $this->oDB->getRow($sSQL, null, 'Could not get outline');
 

lib-php/template/address-geocodejson.php

@@ -36,9 +36,6 @@ if (empty($aPlace)) {
         $aFilteredPlaces['properties']['geocoding']['osm_id'] = $aPlace['osm_id'];
     }
 
-    $aFilteredPlaces['properties']['geocoding']['osm_key'] = $aPlace['class'];
-    $aFilteredPlaces['properties']['geocoding']['osm_value'] = $aPlace['type'];
-
     $aFilteredPlaces['properties']['geocoding']['type'] = addressRankToGeocodeJsonType($aPlace['rank_address']);
 
     $aFilteredPlaces['properties']['geocoding']['accuracy'] = (int) $fDistance;

lib-sql/functions/interpolation.sql

@@ -164,7 +164,7 @@ DECLARE
   newend INTEGER;
   moddiff SMALLINT;
   linegeo GEOMETRY;
-  splitpoint FLOAT;
+  splitline GEOMETRY;
   sectiongeo GEOMETRY;
   postcode TEXT;
   stepmod SMALLINT;
@@ -223,27 +223,15 @@ BEGIN
         FROM placex, generate_series(1, array_upper(waynodes, 1)) nodeidpos
         WHERE osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT
               and address is not NULL and address ? 'housenumber'
-              and ST_Distance(NEW.linegeo, geometry) < 0.0005
         ORDER BY nodeidpos
     LOOP
       {% if debug %}RAISE WARNING 'processing point % (%)', nextnode.hnr, ST_AsText(nextnode.geometry);{% endif %}
       IF linegeo is null THEN
         linegeo := NEW.linegeo;
       ELSE
-        splitpoint := ST_LineLocatePoint(linegeo, nextnode.geometry);
-        IF splitpoint = 0 THEN
-          -- Corner case where the splitpoint falls on the first point
-          -- and thus would not return a geometry. Skip that section.
-          sectiongeo := NULL;
-        ELSEIF splitpoint = 1 THEN
-          -- Point is at the end of the line.
-          sectiongeo := linegeo;
-          linegeo := NULL;
-        ELSE
-          -- Split the line.
-          sectiongeo := ST_LineSubstring(linegeo, 0, splitpoint);
-          linegeo := ST_LineSubstring(linegeo, splitpoint, 1);
-        END IF;
+        splitline := ST_Split(ST_Snap(linegeo, nextnode.geometry, 0.0005), nextnode.geometry);
+        sectiongeo := ST_GeometryN(splitline, 1);
+        linegeo := ST_GeometryN(splitline, 2);
       END IF;
 
       IF prevnode.hnr is not null
@@ -251,9 +239,6 @@ BEGIN
          -- regularly mapped housenumbers.
          -- (Conveniently also fails if one of the house numbers is not a number.)
          and abs(prevnode.hnr - nextnode.hnr) > NEW.step
-         -- If the interpolation geometry is broken or two nodes are at the
-         -- same place, then splitting might produce a point. Ignore that.
-         and ST_GeometryType(sectiongeo) = 'ST_LineString'
       THEN
         IF prevnode.hnr < nextnode.hnr THEN
           startnumber := prevnode.hnr;
@@ -315,12 +300,12 @@ BEGIN
                   NEW.address, postcode,
                   NEW.country_code, NEW.geometry_sector, 0);
         END IF;
-      END IF;
 
-      -- early break if we are out of line string,
-      -- might happen when a line string loops back on itself
-      IF linegeo is null or ST_GeometryType(linegeo) != 'ST_LineString' THEN
-          RETURN NEW;
+        -- early break if we are out of line string,
+        -- might happen when a line string loops back on itself
+        IF ST_GeometryType(linegeo) != 'ST_LineString' THEN
+            RETURN NEW;
+        END IF;
       END IF;
 
       prevnode := nextnode;

lib-sql/functions/place_triggers.sql

@@ -384,19 +384,7 @@ BEGIN
 
   -- Mark for delete in the placex table
   UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
-    WHERE placex.osm_type = 'N' and place_to_be_deleted.osm_type = 'N'
-          and placex.osm_id = place_to_be_deleted.osm_id
-          and placex.class = place_to_be_deleted.class
-          and placex.type = place_to_be_deleted.type
-          and not deferred;
-  UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
-    WHERE placex.osm_type = 'W' and place_to_be_deleted.osm_type = 'W'
-          and placex.osm_id = place_to_be_deleted.osm_id
-          and placex.class = place_to_be_deleted.class
-          and placex.type = place_to_be_deleted.type
-          and not deferred;
-  UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
-    WHERE placex.osm_type = 'R' and place_to_be_deleted.osm_type = 'R'
+    WHERE placex.osm_type = place_to_be_deleted.osm_type
           and placex.osm_id = place_to_be_deleted.osm_id
           and placex.class = place_to_be_deleted.class
           and placex.type = place_to_be_deleted.type

lib-sql/functions/placex_triggers.sql

@@ -1120,7 +1120,7 @@ BEGIN
   ELSE
     -- No linked place? As a last resort check if the boundary is tagged with
     -- a place type and adapt the rank address.
-    IF NEW.rank_address between 4 and 25 and NEW.extratags ? 'place' THEN
+    IF NEW.rank_address > 0 and NEW.extratags ? 'place' THEN
       SELECT address_rank INTO place_address_level
         FROM compute_place_rank(NEW.country_code, 'A', 'place',
                                 NEW.extratags->'place', 0::SMALLINT, False, null);
@@ -1230,11 +1230,7 @@ BEGIN
     {% endif %}
   END IF;
 
-  IF NEW.postcode is null AND NEW.rank_search > 8
-     AND (NEW.rank_address > 0
-          OR ST_GeometryType(NEW.geometry) not in ('ST_LineString','ST_MultiLineString')
-          OR ST_Length(NEW.geometry) < 0.02)
-  THEN
+  IF NEW.postcode is null AND NEW.rank_search > 8 THEN
     NEW.postcode := get_nearest_postcode(NEW.country_code, NEW.geometry);
   END IF;
 

lib-sql/functions/utils.sql

@@ -273,8 +273,8 @@ BEGIN
   END IF;
 
   RETURN ST_Envelope(ST_Collect(
-                     ST_Project(geom::geography, radius, 0.785398)::geometry,
-                     ST_Project(geom::geography, radius, 3.9269908)::geometry));
+                     ST_Project(geom, radius, 0.785398)::geometry,
+                     ST_Project(geom, radius, 3.9269908)::geometry));
 END;
 $$
 LANGUAGE plpgsql IMMUTABLE;
@@ -429,10 +429,9 @@ BEGIN
   SELECT osm_type, osm_id, class, type FROM placex WHERE place_id = placeid INTO osmtype, osmid, pclass, ptype;
   DELETE FROM import_polygon_delete where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
   DELETE FROM import_polygon_error where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
-  -- force delete by directly entering it into the to-be-deleted table
-  INSERT INTO place_to_be_deleted (osm_type, osm_id, class, type, deferred)
-         VALUES(osmtype, osmid, pclass, ptype, false);
-  PERFORM flush_deleted_places();
+  -- force delete from place/placex by making it a very small geometry
+  UPDATE place set geometry = ST_SetSRID(ST_Point(0,0), 4326) where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
+  DELETE FROM place where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
 
   RETURN TRUE;
 END;

nominatim/clicmd/add_data.py

@@ -76,25 +76,21 @@ class UpdateAddData:
 
         osm2pgsql_params = args.osm2pgsql_options(default_cache=1000, default_threads=1)
         if args.file or args.diff:
-            return add_osm_data.add_data_from_file(args.config.get_libpq_dsn(),
-                                                   cast(str, args.file or args.diff),
+            return add_osm_data.add_data_from_file(cast(str, args.file or args.diff),
                                                    osm2pgsql_params)
 
         if args.node:
-            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
-                                               'node', args.node,
+            return add_osm_data.add_osm_object('node', args.node,
                                                args.use_main_api,
                                                osm2pgsql_params)
 
         if args.way:
-            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
-                                               'way', args.way,
+            return add_osm_data.add_osm_object('way', args.way,
                                                args.use_main_api,
                                                osm2pgsql_params)
 
         if args.relation:
-            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
-                                               'relation', args.relation,
+            return add_osm_data.add_osm_object('relation', args.relation,
                                                args.use_main_api,
                                                osm2pgsql_params)
 

nominatim/clicmd/api.py

@@ -248,9 +248,9 @@ class APIDetails:
         if args.node:
             params = dict(osmtype='N', osmid=args.node)
         elif args.way:
-            params = dict(osmtype='W', osmid=args.way)
+            params = dict(osmtype='W', osmid=args.node)
         elif args.relation:
-            params = dict(osmtype='R', osmid=args.relation)
+            params = dict(osmtype='R', osmid=args.node)
         else:
             params = dict(place_id=args.place_id)
         if args.object_class:

nominatim/db/async_connection.py

@@ -69,8 +69,8 @@ class DBConnection:
         self.current_params: Optional[Sequence[Any]] = None
         self.ignore_sql_errors = ignore_sql_errors
 
-        self.conn: Optional['psycopg2._psycopg.connection'] = None
-        self.cursor: Optional['psycopg2._psycopg.cursor'] = None
+        self.conn: Optional['psycopg2.connection'] = None
+        self.cursor: Optional['psycopg2.cursor'] = None
         self.connect(cursor_factory=cursor_factory)
 
     def close(self) -> None:
@@ -78,7 +78,7 @@ class DBConnection:
         """
         if self.conn is not None:
             if self.cursor is not None:
-                self.cursor.close()
+                self.cursor.close() # type: ignore[no-untyped-call]
                 self.cursor = None
             self.conn.close()
 

nominatim/db/connection.py

@@ -31,7 +31,7 @@ class Cursor(psycopg2.extras.DictCursor):
         """ Query execution that logs the SQL query when debugging is enabled.
         """
         if LOG.isEnabledFor(logging.DEBUG):
-            LOG.debug(self.mogrify(query, args).decode('utf-8'))
+            LOG.debug(self.mogrify(query, args).decode('utf-8')) # type: ignore[no-untyped-call]
 
         super().execute(query, args)
 

nominatim/db/utils.py

@@ -118,4 +118,4 @@ class CopyBuffer:
         """
         if self.buffer.tell() > 0:
             self.buffer.seek(0)
-            cur.copy_from(self.buffer, table, columns=columns)
+            cur.copy_from(self.buffer, table, columns=columns) # type: ignore[no-untyped-call]

nominatim/tools/add_osm_data.py

@@ -12,34 +12,23 @@ from pathlib import Path
 import logging
 import urllib
 
-from nominatim.db.connection import connect
 from nominatim.tools.exec_utils import run_osm2pgsql, get_url
 
 LOG = logging.getLogger()
 
-def _run_osm2pgsql(dsn: str, options: MutableMapping[str, Any]) -> None:
-    run_osm2pgsql(options)
-
-    # Handle deletions
-    with connect(dsn) as conn:
-        with conn.cursor() as cur:
-            cur.execute('SELECT flush_deleted_places()')
-        conn.commit()
-
-
-def add_data_from_file(dsn: str, fname: str, options: MutableMapping[str, Any]) -> int:
+def add_data_from_file(fname: str, options: MutableMapping[str, Any]) -> int:
     """ Adds data from a OSM file to the database. The file may be a normal
         OSM file or a diff file in all formats supported by libosmium.
     """
     options['import_file'] = Path(fname)
     options['append'] = True
-    _run_osm2pgsql(dsn, options)
+    run_osm2pgsql(options)
 
     # No status update. We don't know where the file came from.
     return 0
 
 
-def add_osm_object(dsn: str, osm_type: str, osm_id: int, use_main_api: bool,
+def add_osm_object(osm_type: str, osm_id: int, use_main_api: bool,
                    options: MutableMapping[str, Any]) -> int:
     """ Add or update a single OSM object from the latest version of the
         API.
@@ -62,6 +51,6 @@ def add_osm_object(dsn: str, osm_type: str, osm_id: int, use_main_api: bool,
     options['append'] = True
     options['import_data'] = get_url(base_url).encode('utf-8')
 
-    _run_osm2pgsql(dsn, options)
+    run_osm2pgsql(options)
 
     return 0

nominatim/tools/collect_os_info.py

@@ -12,13 +12,14 @@ import os
 import subprocess
 import sys
 from pathlib import Path
-from typing import List, Optional, Tuple, Union
+from typing import List, Optional, Tuple, Union, cast
 
 import psutil
 from psycopg2.extensions import make_dsn, parse_dsn
 
 from nominatim.config import Configuration
 from nominatim.db.connection import connect
+from nominatim.typing import DictCursorResults
 from nominatim.version import version_str
 
 
@@ -106,15 +107,15 @@ def report_system_information(config: Configuration) -> None:
         postgresql_ver: str = convert_version(conn.server_version_tuple())
 
         with conn.cursor() as cur:
-            num = cur.scalar("SELECT count(*) FROM pg_catalog.pg_database WHERE datname=%s",
-                             (parse_dsn(config.get_libpq_dsn())['dbname'], ))
-            nominatim_db_exists = num == 1 if isinstance(num, int) else False
-
-    if nominatim_db_exists:
-        with connect(config.get_libpq_dsn()) as conn:
-            postgis_ver: str = convert_version(conn.postgis_version_tuple())
-    else:
-        postgis_ver = "Unable to connect to database"
+            cur.execute(f"""
+            SELECT datname FROM pg_catalog.pg_database 
+            WHERE datname='{parse_dsn(config.get_libpq_dsn())['dbname']}'""")
+            nominatim_db_exists = cast(Optional[DictCursorResults], cur.fetchall())
+            if nominatim_db_exists:
+                with connect(config.get_libpq_dsn()) as conn:
+                    postgis_ver: str = convert_version(conn.postgis_version_tuple())
+            else:
+                postgis_ver = "Unable to connect to database"
 
     postgresql_config: str = get_postgresql_config(int(float(postgresql_ver)))
 

nominatim/tools/exec_utils.py

@@ -118,7 +118,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
     cmd = [str(options['osm2pgsql']),
            '--hstore', '--latlon', '--slim',
            '--log-progress', 'true',
-           '--number-processes', '1' if options['append'] else str(options['threads']),
+           '--number-processes', str(options['threads']),
            '--cache', str(options['osm2pgsql_cache']),
            '--style', str(options['osm2pgsql_style'])
           ]

nominatim/tools/refresh.py

@@ -176,7 +176,7 @@ def recompute_importance(conn: Connection) -> None:
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance
-                FROM compute_importance(extratags, country_code, rank_search, centroid))
+                FROM compute_importance(extratags, country_code, osm_type, osm_id, centroid))
             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance

nominatim/version.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2023 by the Nominatim developer community.
+# Copyright (C) 2022 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Version information for Nominatim.
@@ -25,7 +25,7 @@ from typing import Optional, Tuple
 # patch level when cherry-picking the commit with the migration.
 #
 # Released versions always have a database patch level of 0.
-NOMINATIM_VERSION = (4, 2, 4, 0)
+NOMINATIM_VERSION = (4, 2, 0, 0)
 
 POSTGRESQL_REQUIRED_VERSION = (9, 6)
 POSTGIS_REQUIRED_VERSION = (2, 2)

settings/flex-base.lua

@@ -1,19 +1,9 @@
 -- Core functions for Nominatim import flex style.
 --
 
-local module = {}
-
-local PRE_DELETE = nil
-local PRE_EXTRAS = nil
-local MAIN_KEYS = nil
-local NAMES = nil
-local ADDRESS_TAGS = nil
-local SAVE_EXTRA_MAINS = false
-local POSTCODE_FALLBACK = true
-
 
 -- The single place table.
-local place_table = osm2pgsql.define_table{
+place_table = osm2pgsql.define_table{
     name = "place",
     ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
     columns = {
@@ -24,25 +14,7 @@ local place_table = osm2pgsql.define_table{
         { column = 'address', type = 'hstore' },
         { column = 'extratags', type = 'hstore' },
         { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
-    },
-    indexes = {}
-}
-
------------- Geometry functions for relations ---------------------
-
-function module.relation_as_multipolygon(o)
-    return o:as_multipolygon()
-end
-
-function module.relation_as_multiline(o)
-    return o:as_multilinestring():line_merge()
-end
-
-
-module.RELATION_TYPES = {
-    multipolygon = module.relation_as_multipolygon,
-    boundary = module.relation_as_multipolygon,
-    waterway = module.relation_as_multiline
+    }
 }
 
 ------------- Place class ------------------------------------------
@@ -71,17 +43,6 @@ function Place.new(object, geom_func)
     return self
 end
 
-function Place:clean(data)
-    for k, v in pairs(self.object.tags) do
-        if data.delete ~= nil and data.delete(k, v) then
-            self.object.tags[k] = nil
-        elseif data.extra ~= nil and data.extra(k, v) then
-            self.extratags[k] = v
-            self.object.tags[k] = nil
-        end
-    end
-end
-
 function Place:delete(data)
     if data.match ~= nil then
         for k, v in pairs(self.object.tags) do
@@ -108,37 +69,31 @@ function Place:grab_extratags(data)
     return count
 end
 
-local function strip_address_prefix(k)
-    if k:sub(1, 5) == 'addr:' then
-        return k:sub(6)
-    end
-
-    if k:sub(1, 6) == 'is_in:' then
-        return k:sub(7)
-    end
-
-    return k
-end
-
-
-function Place:grab_address_parts(data)
+function Place:grab_address(data)
     local count = 0
 
-    if data.groups ~= nil then
+    if data.match ~= nil then
         for k, v in pairs(self.object.tags) do
-            local atype = data.groups(k, v)
-
-            if atype ~= nil then
-                if atype == 'main' then
-                    self.has_name = true
-                    self.address[strip_address_prefix(k)] = v
-                    count = count + 1
-                elseif atype == 'extra' then
-                    self.address[strip_address_prefix(k)] = v
-                else
-                    self.address[atype] = v
-                end
+            if data.match(k, v) then
                 self.object.tags[k] = nil
+
+                if data.include_on_name == true then
+                    self.has_name = true
+                end
+
+                if data.out_key ~= nil then
+                    self.address[data.out_key] = v
+                    return 1
+                end
+
+                if k:sub(1, 5) == 'addr:' then
+                    self.address[k:sub(6)] = v
+                elseif k:sub(1, 6) == 'is_in:' then
+                    self.address[k:sub(7)] = v
+                else
+                    self.address[k] = v
+                end
+                count = count + 1
             end
         end
     end
@@ -146,30 +101,36 @@ function Place:grab_address_parts(data)
     return count
 end
 
+function Place:set_address(key, value)
+    self.address[key] = value
+end
 
-function Place:grab_name_parts(data)
-    local fallback = nil
+function Place:grab_name(data)
+    local count = 0
 
-    if data.groups ~= nil then
+    if data.match ~= nil then
         for k, v in pairs(self.object.tags) do
-            local atype = data.groups(k, v)
-
-            if atype ~= nil then
-                self.names[k] = v
+            if data.match(k, v) then
                 self.object.tags[k] = nil
-                if atype == 'main' then
+                self.names[k] = v
+                if data.include_on_name ~= false then
                     self.has_name = true
-                elseif atype == 'house' then
-                    self.has_name = true
-                    fallback = {'place', 'house', 'always'}
                 end
+                count = count + 1
             end
         end
     end
 
-    return fallback
+    return count
 end
 
+function Place:grab_tag(key)
+    return self.object:grab_tag(key)
+end
+
+function Place:tags()
+    return self.object.tags
+end
 
 function Place:write_place(k, v, mtype, save_extra_mains)
     if mtype == nil then
@@ -223,9 +184,9 @@ function Place:write_row(k, v, save_extra_mains)
         return 0
     end
 
-    if save_extra_mains ~= nil then
+    if save_extra_mains then
         for extra_k, extra_v in pairs(self.object.tags) do
-            if extra_k ~= k and save_extra_mains(extra_k, extra_v) then
+            if extra_k ~= k then
                 self.extratags[extra_k] = extra_v
             end
         end
@@ -243,9 +204,7 @@ function Place:write_row(k, v, save_extra_mains)
 
     if save_extra_mains then
         for k, v in pairs(self.object.tags) do
-            if save_extra_mains(k, v) then
-                self.extratags[k] = nil
-            end
+            self.extratags[k] = nil
         end
     end
 
@@ -255,7 +214,7 @@ function Place:write_row(k, v, save_extra_mains)
 end
 
 
-function module.tag_match(data)
+function tag_match(data)
     if data == nil or next(data) == nil then
         return nil
     end
@@ -317,72 +276,17 @@ function module.tag_match(data)
 end
 
 
-function module.tag_group(data)
-    if data == nil or next(data) == nil then
-        return nil
-    end
-
-    local fullmatches = {}
-    local key_prefixes = {}
-    local key_suffixes = {}
-
-    for group, tags in pairs(data) do
-        for _, key in pairs(tags) do
-            if key:sub(1, 1) == '*' then
-                if #key > 1 then
-                    if key_suffixes[#key - 1] == nil then
-                        key_suffixes[#key - 1] = {}
-                    end
-                    key_suffixes[#key - 1][key:sub(2)] = group
-                end
-            elseif key:sub(#key, #key) == '*' then
-                if key_prefixes[#key - 1] == nil then
-                    key_prefixes[#key - 1] = {}
-                end
-                key_prefixes[#key - 1][key:sub(1, #key - 1)] = group
-            else
-                fullmatches[key] = group
-            end
-        end
-    end
-
-    return function (k, v)
-        local val = fullmatches[k]
-        if val ~= nil then
-            return val
-        end
-
-        for slen, slist in pairs(key_suffixes) do
-            if #k >= slen then
-                val = slist[k:sub(-slen)]
-                if val ~= nil then
-                    return val
-                end
-            end
-        end
-
-        for slen, slist in pairs(key_prefixes) do
-            if #k >= slen then
-                val = slist[k:sub(1, slen)]
-                if val ~= nil then
-                    return val
-                end
-            end
-        end
-    end
-end
-
 -- Process functions for all data types
-function module.process_node(object)
+function osm2pgsql.process_node(object)
 
     local function geom_func(o)
         return o:as_point()
     end
 
-    module.process_tags(Place.new(object, geom_func))
+    process_tags(Place.new(object, geom_func))
 end
 
-function module.process_way(object)
+function osm2pgsql.process_way(object)
 
     local function geom_func(o)
         local geom = o:as_polygon()
@@ -394,24 +298,30 @@ function module.process_way(object)
         return geom
     end
 
-    module.process_tags(Place.new(object, geom_func))
+    process_tags(Place.new(object, geom_func))
 end
 
-function module.process_relation(object)
-    local geom_func = module.RELATION_TYPES[object.tags.type]
+function relation_as_multipolygon(o)
+    return o:as_multipolygon()
+end
+
+function relation_as_multiline(o)
+    return o:as_multilinestring():line_merge()
+end
+
+function osm2pgsql.process_relation(object)
+    local geom_func = RELATION_TYPES[object.tags.type]
 
     if geom_func ~= nil then
-        module.process_tags(Place.new(object, geom_func))
+        process_tags(Place.new(object, geom_func))
     end
 end
 
--- The process functions are used by default by osm2pgsql.
-osm2pgsql.process_node = module.process_node
-osm2pgsql.process_way = module.process_way
-osm2pgsql.process_relation = module.process_relation
+function process_tags(o)
+    local fallback
 
-function module.process_tags(o)
-    o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS}
+    o:delete{match = PRE_DELETE}
+    o:grab_extratags{match = PRE_EXTRAS}
 
     -- Exception for boundary/place double tagging
     if o.object.tags.boundary == 'administrative' then
@@ -420,91 +330,54 @@ function module.process_tags(o)
         end}
     end
 
-    -- name keys
-    local fallback = o:grab_name_parts{groups=NAMES}
-
     -- address keys
-    if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then
-        fallback = {'place', 'house', 'always'}
-    end
+    o:grab_address{match=COUNTRY_TAGS, out_key='country'}
     if o.address.country ~= nil and #o.address.country ~= 2 then
         o.address['country'] = nil
     end
-    if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then
-        fallback = {'place', 'postcode', 'always'}
+    if o:grab_name{match=HOUSENAME_TAGS} > 0 then
+        fallback = {'place', 'house'}
+    end
+    if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then
+        fallback = {'place', 'house'}
+    end
+    if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then
+        fallback = {'place', 'postcode'}
     end
 
-    if o.address.interpolation ~= nil then
+    local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0
+
+    o:grab_address{match=ADDRESS_TAGS}
+
+    if is_interpolation then
         o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
         return
     end
 
-    o:clean{delete = POST_DELETE}
+    -- name keys
+    o:grab_name{match = NAMES}
+    o:grab_name{match = REFS, include_on_name = false}
+
+    o:delete{match = POST_DELETE}
+    o:grab_extratags{match = POST_EXTRAS}
 
     -- collect main keys
-    for k, v in pairs(o.object.tags) do
-        local ktype = MAIN_KEYS[k]
-        if ktype == 'fallback' then
-            if o.has_name then
-                fallback = {k, v, 'named'}
+    local num_mains = 0
+    for k, v in pairs(o:tags()) do
+        num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
+    end
+
+    if num_mains == 0 then
+        for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do
+            if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then
+                return
             end
-        elseif ktype ~= nil then
-            o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
         end
-    end
 
-    if fallback ~= nil and o.num_entries == 0 then
-        o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS)
-    end
-end
-
---------- Convenience functions for simple style configuration -----------------
-
-
-function module.set_prefilters(data)
-    PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
-    PRE_EXTRAS = module.tag_match{keys = data.extratag_keys,
-                                  tags = data.extratag_tags}
-end
-
-function module.set_main_tags(data)
-    MAIN_KEYS = data
-end
-
-function module.set_name_tags(data)
-    NAMES = module.tag_group(data)
-end
-
-function module.set_address_tags(data)
-    if data.postcode_fallback ~= nil then
-        POSTCODE_FALLBACK = data.postcode_fallback
-        data.postcode_fallback = nil
-    end
-
-    ADDRESS_TAGS = module.tag_group(data)
-end
-
-function module.set_unused_handling(data)
-    if data.extra_keys == nil and data.extra_tags == nil then
-        POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
-        SAVE_EXTRA_MAINS = function() return true end
-    elseif data.delete_keys == nil and data.delete_tags == nil then
-        POST_DELETE = nil
-        SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags}
-    else
-        error("unused handler can have only 'extra_keys' or 'delete_keys' set.")
-    end
-end
-
-function set_relation_types(data)
-    module.RELATION_TYPES = {}
-    for k, v in data do
-        if v == 'multipolygon' then
-            module.RELATION_TYPES[k] = module.relation_as_multipolygon
-        elseif v == 'multiline' then
-            module.RELATION_TYPES[k] = module.relation_as_multiline
+        if fallback ~= nil then
+            o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS)
         end
     end
 end
 
-return module
+

settings/icu_tokenizer.yaml

@@ -24,7 +24,6 @@ transliteration:
     - ":: lower ()"
     - "[^a-z0-9[:Space:]] >"
     - ":: NFC ()"
-    - "[:Space:]+ > ' '"
 sanitizers:
     - step: clean-housenumbers
       filter-kind:
@@ -38,7 +37,6 @@ sanitizers:
       default-pattern: "[A-Z0-9- ]{3,12}"
     - step: clean-tiger-tags
     - step: split-name-list
-      delimiters: ;
     - step: strip-brace-terms
     - step: tag-analyzer-by-language
       filter-kind: [".*name.*"]

settings/import-address.lua

@@ -1,67 +0,0 @@
-flex = require('flex-base')
-
-flex.set_main_tags{
-    highway = {'always',
-               street_lamp = 'named',
-               traffic_signals = 'named',
-               service = 'named',
-               cycleway = 'named',
-               path = 'named',
-               footway = 'named',
-               steps = 'named',
-               bridleway = 'named',
-               track = 'named',
-               motorway_link = 'named',
-               trunk_link = 'named',
-               primary_link = 'named',
-               secondary_link = 'named',
-               tertiary_link = 'named'},
-    boundary = {administrative = 'named',
-                postal_code = 'always'},
-    landuse = 'fallback',
-    place = 'always'
-}
-
-flex.set_prefilters{delete_keys = {'building', 'source',
-                                   'source', '*source', 'type',
-                                   'is_in:postcode', '*:wikidata', '*:wikipedia',
-                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
-                                   'name:etymology', 'name:signed', 'name:botanical',
-                                   'addr:street:name', 'addr:street:type'},
-                    delete_tags = {highway = {'no', 'turning_circle', 'mini_roundabout',
-                                              'noexit', 'crossing', 'give_way', 'stop'},
-                                   landuse = {'cemetry', 'no'},
-                                   boundary = {'place'}},
-                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
-                   }
-
-flex.set_name_tags{main = {'name', 'name:*',
-                          'int_name', 'int_name:*',
-                          'nat_name', 'nat_name:*',
-                          'reg_name', 'reg_name:*',
-                          'loc_name', 'loc_name:*',
-                          'old_name', 'old_name:*',
-                          'alt_name', 'alt_name:*', 'alt_name_*',
-                          'official_name', 'official_name:*',
-                          'place_name', 'place_name:*',
-                          'short_name', 'short_name:*'},
-                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
-                            'loc_ref', 'old_ref',
-                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'},
-                   house = {'addr:housename'}
-                  }
-
-flex.set_address_tags{main = {'addr:housenumber',
-                              'addr:conscriptionnumber',
-                              'addr:streetnumber'},
-                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
-                      postcode = {'postal_code', 'postcode', 'addr:postcode',
-                                  'tiger:zip_left', 'tiger:zip_right'},
-                      country = {'country_code', 'ISO3166-1',
-                                 'addr:country_code', 'is_in:country_code',
-                                 'addr:country', 'is_in:country'},
-                      interpolation = {'addr:interpolation'}
-                     }
-
-
-flex.set_unused_handling{extra_keys = {'place'}}

settings/import-admin.lua

@@ -1,44 +0,0 @@
-local flex = require('flex-base')
-
-flex.set_main_tags{
-    boundary = {administrative = 'named'},
-    landuse = 'fallback',
-    place = 'always'
-}
-
-flex.set_prefilters{delete_keys = {'building', 'source', 'highway',
-                                   'addr:housenumber', 'addr:street', 'addr:city',
-                                   'source', '*source', 'type',
-                                   'is_in:postcode', '*:wikidata', '*:wikipedia',
-                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
-                                   'name:etymology', 'name:signed', 'name:botanical',
-                                   'addr:street:name', 'addr:street:type'},
-                    delete_tags = {landuse = {'cemetry', 'no'},
-                                   boundary = {'place'}},
-                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
-                   }
-
-flex.set_name_tags{main = {'name', 'name:*',
-                          'int_name', 'int_name:*',
-                          'nat_name', 'nat_name:*',
-                          'reg_name', 'reg_name:*',
-                          'loc_name', 'loc_name:*',
-                          'old_name', 'old_name:*',
-                          'alt_name', 'alt_name:*', 'alt_name_*',
-                          'official_name', 'official_name:*',
-                          'place_name', 'place_name:*',
-                          'short_name', 'short_name:*'},
-                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
-                            'loc_ref', 'old_ref',
-                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}
-                  }
-
-flex.set_address_tags{extra = {'addr:*', 'is_in:*'},
-                      postcode = {'postal_code', 'postcode', 'addr:postcode'},
-                      country = {'country_code', 'ISO3166-1',
-                                 'addr:country_code', 'is_in:country_code',
-                                 'addr:country', 'is_in:country'},
-                      postcode_fallback = false
-                     }
-
-flex.set_unused_handling{extra_keys = {'place'}}

settings/import-extratags.lua

@@ -1,9 +1,13 @@
-flex = require('flex-base')
+require('flex-base')
 
-flex.set_main_tags{
-    building = 'fallback',
+RELATION_TYPES = {
+    multipolygon = relation_as_multipolygon,
+    boundary = relation_as_multipolygon,
+    waterway = relation_as_multiline
+}
+
+MAIN_KEYS = {
     emergency = 'always',
-    healthcare = 'fallback',
     historic = 'always',
     military = 'always',
     natural = 'named',
@@ -27,13 +31,11 @@ flex.set_main_tags{
     man_made = 'always',
     aerialway = 'always',
     boundary = {'named',
-                postal_code = 'always'},
+                postal_code = 'named'},
     aeroway = 'always',
     amenity = 'always',
     club = 'always',
     craft = 'always',
-    junction = 'fallback',
-    landuse = 'fallback',
     leisure = 'always',
     office = 'always',
     mountain_pass = 'always',
@@ -45,43 +47,55 @@ flex.set_main_tags{
     place = 'always'
 }
 
-flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attribution',
-                                   'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
-                                   'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
-                                   'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
-                                   'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
-                                   'ref:linz:*', 'is_in:postcode'},
-                    delete_tags = {emergency = {'yes', 'no', 'fire_hydrant'},
-                                   historic = {'yes', 'no'},
-                                   military = {'yes', 'no'},
-                                   natural = {'yes', 'no', 'coastline'},
-                                   highway = {'no', 'turning_circle', 'mini_roundabout',
-                                              'noexit', 'crossing', 'give_way', 'stop'},
-                                   railway = {'level_crossing', 'no', 'rail'},
-                                   man_made = {'survey_point', 'cutline'},
-                                   aerialway = {'pylon', 'no'},
-                                   aeroway = {'no'},
-                                   amenity = {'no'},
-                                   club = {'no'},
-                                   craft = {'no'},
-                                   leisure = {'no'},
-                                   office = {'no'},
-                                   mountain_pass = {'no'},
-                                   shop = {'no'},
-                                   tourism = {'yes', 'no'},
-                                   bridge = {'no'},
-                                   tunnel = {'no'},
-                                   waterway = {'riverbank'},
-                                   building = {'no'},
-                                   boundary = {'place'}},
-                    extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+MAIN_FALLBACK_KEYS = {
+    building = 'named',
+    landuse = 'named',
+    junction = 'named',
+    healthcare = 'named'
+}
+
+
+PRE_DELETE = tag_match{keys = {'note', 'note:*', 'source', 'source*', 'attribution',
+                               'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
+                               'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
+                               'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
+                               'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
+                               'ref:linz:*', 'is_in:postcode'},
+                       tags = {emergency = {'yes', 'no', 'fire_hydrant'},
+                               historic = {'yes', 'no'},
+                               military = {'yes', 'no'},
+                               natural = {'yes', 'no', 'coastline'},
+                               highway = {'no', 'turning_circle', 'mini_roundabout',
+                                          'noexit', 'crossing', 'give_way', 'stop'},
+                               railway = {'level_crossing', 'no', 'rail'},
+                               man_made = {'survey_point', 'cutline'},
+                               aerialway = {'pylon', 'no'},
+                               aeroway = {'no'},
+                               amenity = {'no'},
+                               club = {'no'},
+                               craft = {'no'},
+                               leisure = {'no'},
+                               office = {'no'},
+                               mountain_pass = {'no'},
+                               shop = {'no'},
+                               tourism = {'yes', 'no'},
+                               bridge = {'no'},
+                               tunnel = {'no'},
+                               waterway = {'riverbank'},
+                               building = {'no'},
+                               boundary = {'place'}}
+                      }
+
+POST_DELETE = tag_match{keys = {'tiger:*'}}
+
+PRE_EXTRAS = tag_match{keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
                                'name:etymology', 'name:signed', 'name:botanical',
                                'wikidata', '*:wikidata',
-                               '*:wikipedia', 'brand:wikipedia:*',
                                'addr:street:name', 'addr:street:type'}
-                   }
+                      }
 
-flex.set_name_tags{main = {'name', 'name:*',
+
+NAMES = tag_match{keys = {'name', 'name:*',
                           'int_name', 'int_name:*',
                           'nat_name', 'nat_name:*',
                           'reg_name', 'reg_name:*',
@@ -90,24 +104,26 @@ flex.set_name_tags{main = {'name', 'name:*',
                           'alt_name', 'alt_name:*', 'alt_name_*',
                           'official_name', 'official_name:*',
                           'place_name', 'place_name:*',
-                          'short_name', 'short_name:*', 'brand'},
-                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
-                            'loc_ref', 'old_ref',
-                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'},
-                   house = {'addr:housename'}
-                  }
+                          'short_name', 'short_name:*', 'brand'}}
 
-flex.set_address_tags{main = {'addr:housenumber',
-                              'addr:conscriptionnumber',
-                              'addr:streetnumber'},
-                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
-                      postcode = {'postal_code', 'postcode', 'addr:postcode',
-                                  'tiger:zip_left', 'tiger:zip_right'},
-                      country = {'country_code', 'ISO3166-1',
+REFS = tag_match{keys = {'ref', 'int_ref', 'nat_ref', 'reg_ref', 'loc_ref', 'old_ref',
+                         'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}}
+
+POSTCODES = tag_match{keys = {'postal_code', 'postcode', 'addr:postcode',
+                              'tiger:zip_left', 'tiger:zip_right'}}
+
+COUNTRY_TAGS = tag_match{keys = {'country_code', 'ISO3166-1',
                                  'addr:country_code', 'is_in:country_code',
-                                 'addr:country', 'is_in:country'},
-                      interpolation = {'addr:interpolation'}
-                     }
+                                 'addr:country', 'is_in:country'}}
 
+HOUSENAME_TAGS = tag_match{keys = {'addr:housename'}}
+
+HOUSENUMBER_TAGS = tag_match{keys = {'addr:housenumber', 'addr:conscriptionnumber',
+                                     'addr:streetnumber'}}
+
+INTERPOLATION_TAGS = tag_match{keys = {'addr:interpolation'}}
+
+ADDRESS_TAGS = tag_match{keys = {'addr:*', 'is_in:*', 'tiger:county'}}
+
+SAVE_EXTRA_MAINS = true
 
-flex.set_unused_handling{delete_keys = {'tiger:*'}}

settings/import-full.lua

@@ -1,113 +0,0 @@
-flex = require('flex-base')
-
-flex.set_main_tags{
-    building = 'fallback',
-    emergency = 'always',
-    healthcare = 'fallback',
-    historic = 'always',
-    military = 'always',
-    natural = 'named',
-    landuse = 'named',
-    highway = {'always',
-               street_lamp = 'named',
-               traffic_signals = 'named',
-               service = 'named',
-               cycleway = 'named',
-               path = 'named',
-               footway = 'named',
-               steps = 'named',
-               bridleway = 'named',
-               track = 'named',
-               motorway_link = 'named',
-               trunk_link = 'named',
-               primary_link = 'named',
-               secondary_link = 'named',
-               tertiary_link = 'named'},
-    railway = 'named',
-    man_made = 'always',
-    aerialway = 'always',
-    boundary = {'named',
-                postal_code = 'always'},
-    aeroway = 'always',
-    amenity = 'always',
-    club = 'always',
-    craft = 'always',
-    junction = 'fallback',
-    landuse = 'fallback',
-    leisure = 'always',
-    office = 'always',
-    mountain_pass = 'always',
-    shop = 'always',
-    tourism = 'always',
-    bridge = 'named_with_key',
-    tunnel = 'named_with_key',
-    waterway = 'named',
-    place = 'always'
-}
-
-flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attribution',
-                                   'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
-                                   'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
-                                   'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
-                                   'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
-                                   'ref:linz:*', 'is_in:postcode',
-                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
-                                   'name:etymology', 'name:signed', 'name:botanical',
-                                   '*:wikidata', '*:wikipedia', 'brand:wikipedia:*',
-                                   'addr:street:name', 'addr:street:type'},
-                    delete_tags = {emergency = {'yes', 'no', 'fire_hydrant'},
-                                   historic = {'yes', 'no'},
-                                   military = {'yes', 'no'},
-                                   natural = {'yes', 'no', 'coastline'},
-                                   highway = {'no', 'turning_circle', 'mini_roundabout',
-                                              'noexit', 'crossing', 'give_way', 'stop'},
-                                   railway = {'level_crossing', 'no', 'rail'},
-                                   man_made = {'survey_point', 'cutline'},
-                                   aerialway = {'pylon', 'no'},
-                                   aeroway = {'no'},
-                                   amenity = {'no'},
-                                   club = {'no'},
-                                   craft = {'no'},
-                                   leisure = {'no'},
-                                   office = {'no'},
-                                   mountain_pass = {'no'},
-                                   shop = {'no'},
-                                   tourism = {'yes', 'no'},
-                                   bridge = {'no'},
-                                   tunnel = {'no'},
-                                   waterway = {'riverbank'},
-                                   building = {'no'},
-                                   boundary = {'place'}},
-                    extra_keys = {'wikidata', 'wikipedia', 'wikipedia:*'}
-                   }
-
-flex.set_name_tags{main = {'name', 'name:*',
-                          'int_name', 'int_name:*',
-                          'nat_name', 'nat_name:*',
-                          'reg_name', 'reg_name:*',
-                          'loc_name', 'loc_name:*',
-                          'old_name', 'old_name:*',
-                          'alt_name', 'alt_name:*', 'alt_name_*',
-                          'official_name', 'official_name:*',
-                          'place_name', 'place_name:*',
-                          'short_name', 'short_name:*', 'brand'},
-                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
-                            'loc_ref', 'old_ref',
-                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'},
-                   house = {'addr:housename'}
-                  }
-
-flex.set_address_tags{main = {'addr:housenumber',
-                              'addr:conscriptionnumber',
-                              'addr:streetnumber'},
-                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
-                      postcode = {'postal_code', 'postcode', 'addr:postcode',
-                                  'tiger:zip_left', 'tiger:zip_right'},
-                      country = {'country_code', 'ISO3166-1',
-                                 'addr:country_code', 'is_in:country_code',
-                                 'addr:country', 'is_in:country'},
-                      interpolation = {'addr:interpolation'}
-                     }
-
-
-flex.set_unused_handling{extra_keys = {'place'}}

settings/import-street.lua

@@ -1,67 +0,0 @@
-flex = require('flex-base')
-
-flex.set_main_tags{
-    highway = {'always',
-               street_lamp = 'named',
-               traffic_signals = 'named',
-               service = 'named',
-               cycleway = 'named',
-               path = 'named',
-               footway = 'named',
-               steps = 'named',
-               bridleway = 'named',
-               track = 'named',
-               motorway_link = 'named',
-               trunk_link = 'named',
-               primary_link = 'named',
-               secondary_link = 'named',
-               tertiary_link = 'named'},
-    boundary = {administrative = 'named',
-                postal_code = 'always'},
-    landuse = 'fallback',
-    place = 'always'
-}
-
-flex.set_prefilters{delete_keys = {'building', 'source',
-                                   'addr:housenumber', 'addr:street',
-                                   'source', '*source', 'type',
-                                   'is_in:postcode', '*:wikidata', '*:wikipedia',
-                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
-                                   'name:etymology', 'name:signed', 'name:botanical',
-                                   'addr:street:name', 'addr:street:type'},
-                    delete_tags = {highway = {'no', 'turning_circle', 'mini_roundabout',
-                                              'noexit', 'crossing', 'give_way', 'stop'},
-                                   landuse = {'cemetry', 'no'},
-                                   boundary = {'place'}},
-                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
-                   }
-
-flex.set_name_tags{main = {'name', 'name:*',
-                          'int_name', 'int_name:*',
-                          'nat_name', 'nat_name:*',
-                          'reg_name', 'reg_name:*',
-                          'loc_name', 'loc_name:*',
-                          'old_name', 'old_name:*',
-                          'alt_name', 'alt_name:*', 'alt_name_*',
-                          'official_name', 'official_name:*',
-                          'place_name', 'place_name:*',
-                          'short_name', 'short_name:*'},
-                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
-                            'loc_ref', 'old_ref',
-                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}
-                  }
-
-flex.set_address_tags{main = {'addr:housenumber',
-                              'addr:conscriptionnumber',
-                              'addr:streetnumber'},
-                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
-                      postcode = {'postal_code', 'postcode', 'addr:postcode',
-                                  'tiger:zip_left', 'tiger:zip_right'},
-                      country = {'country_code', 'ISO3166-1',
-                                 'addr:country_code', 'is_in:country_code',
-                                 'addr:country', 'is_in:country'},
-                      interpolation = {'addr:interpolation'},
-                      postcode_fallback = false
-                     }
-
-flex.set_unused_handling{extra_keys = {'place'}}

test/Makefile

@@ -1,9 +1,12 @@
-all: bdd php python
+all: bdd php
 no-test-db: bdd-no-test-db php
 
 bdd:
 	cd bdd && behave -DREMOVE_TEMPLATE=1
 
+icu:
+	cd bdd && behave -DREMOVE_TEMPLATE=1 -DTOKENIZER=icu
+
 php:
 	cd php && phpunit ./
 
@@ -11,4 +14,4 @@ python:
 	pytest python
 
 
-.PHONY: bdd php no-test-db python
+.PHONY: bdd php no-test-db

test/bdd/db/import/interpolation.feature

@@ -29,14 +29,14 @@ Feature: Import of address interpolations
           | N2  | place | house  | 8       |
         And the places
           | osm | class | type   | addr+interpolation | geometry |
-          | W1  | place | houses | even               | 2,1      |
+          | W1  | place | houses | even    | 1,2 |
         And the ways
           | id | nodes |
           | 1  | 2,1 |
         When importing
         Then W1 expands to interpolation
           | start | end | geometry |
-          | 4     | 6   | 9,8      |
+          | 4     | 6   | 8,9 |
 
     Scenario: Simple odd two point interpolation
         Given the grid with origin 1,1
@@ -341,7 +341,7 @@ Feature: Import of address interpolations
         Then W1 expands to interpolation
           | start | end | geometry |
           | 4     | 4   | 144.963016 -37.762946 |
-          | 8     | 8   | 144.96314407 -37.762223692 |
+          | 8     | 8   | 144.963144 -37.7622237 |
 
     Scenario: Place with missing address information
         Given the grid
@@ -456,69 +456,3 @@ Feature: Import of address interpolations
           | foo   |
           | x     |
           | 12-2  |
-
-
-    Scenario: Interpolation line where points have been moved (Github #3022)
-        Given the 0.00001 grid
-         | 1 | | | | | | | | 2 | 3 | 9 | | | | | | | | 4 |
-        Given the places
-          | osm | class | type   | housenr | geometry |
-          | N1  | place | house  | 2       | 1 |
-          | N2  | place | house  | 18      | 3 |
-          | N3  | place | house  | 24      | 9 |
-          | N4  | place | house  | 42      | 4 |
-        And the places
-          | osm | class | type   | addr+interpolation | geometry |
-          | W1  | place | houses | even               | 1,2,3,4  |
-        And the ways
-          | id | nodes   |
-          | 1  | 1,2,3,4 |
-        When importing
-        Then W1 expands to interpolation
-          | start | end |
-          | 4     | 16  |
-          | 20    | 22  |
-          | 26    | 40  |
-
-
-    Scenario: Interpolation line with duplicated points
-        Given the grid
-          | 7 | 10 | 8 | 11 | 9 |
-        Given the places
-          | osm | class | type   | housenr | geometry |
-          | N1  | place | house  | 2       | 7 |
-          | N2  | place | house  | 6       | 8 |
-          | N3  | place | house  | 10      | 8 |
-          | N4  | place | house  | 14      | 9 |
-        And the places
-          | osm | class | type   | addr+interpolation | geometry |
-          | W1  | place | houses | even               | 7,8,8,9  |
-        And the ways
-          | id | nodes   |
-          | 1  | 1,2,3,4 |
-        When importing
-        Then W1 expands to interpolation
-          | start | end | geometry |
-          | 4     | 4   | 10       |
-          | 12    | 12  | 11       |
-
-
-    Scenario: Interpolaton line with broken way geometry (Github #2986)
-        Given the grid
-          | 1 | 8 | 10 | 11 | 9 | 2 | 3 | 4 |
-        Given the places
-          | osm | class | type   | housenr |
-          | N1  | place | house  | 2       |
-          | N2  | place | house  | 8       |
-          | N3  | place | house  | 12      |
-          | N4  | place | house  | 14      |
-        And the places
-          | osm | class | type   | addr+interpolation | geometry |
-          | W1  | place | houses | even               | 8,9      |
-        And the ways
-          | id | nodes       |
-          | 1  | 1,8,9,2,3,4 |
-        When importing
-        Then W1 expands to interpolation
-          | start | end | geometry |
-          | 4     | 6   | 10,11    |

test/bdd/db/import/rank_computation.feature

@@ -255,15 +255,3 @@ Feature: Rank assignment
          | W1     | R10     | True      | 18                  |
          | W1     | R2      | True      | 16                  |
          | W1     | N9      | False     | 18                  |
-
-
-    Scenario: POI nodes with place tags
-        Given the places
-          | osm | class   | type       | name | extratags       |
-          | N23 | amenity | playground | AB   | "place": "city" |
-          | N23 | place   | city       | AB   | "amenity": "playground" |
-        When importing
-        Then placex contains exactly
-          | object      | rank_search | rank_address |
-          | N23:amenity | 30          | 30           |
-          | N23:place   | 16          | 16           |

test/bdd/db/import/search_name.feature

@@ -2,26 +2,21 @@
 Feature: Creation of search terms
     Tests that search_name table is filled correctly
 
-    Scenario: Semicolon-separated names appear as separate full names
+    Scenario Outline: Comma- and semicolon separated names appear as full names
         Given the places
          | osm | class   | type | name+alt_name |
-         | N1  | place   | city | New York; Big Apple |
+         | N1  | place   | city | New York<sep>Big Apple |
         When importing
         Then search_name contains
          | object | name_vector |
          | N1     | #New York, #Big Apple |
 
-    @fail-legacy
-    Scenario: Comma-separated names appear as a single full name
-        Given the places
-         | osm | class   | type | name+alt_name |
-         | N1  | place   | city | New York, Big Apple |
-        When importing
-        Then search_name contains
-         | object | name_vector |
-         | N1     | #New York Big Apple |
+    Examples:
+         | sep |
+         | ,   |
+         | ;   |
 
-    Scenario: Name parts before brackets appear as full names
+    Scenario Outline: Name parts before brackets appear as full names
         Given the places
          | osm | class   | type | name+name |
          | N1  | place   | city | Halle (Saale) |

test/bdd/osm2pgsql/import/tags.feature

@@ -101,19 +101,6 @@ Feature: Tag evaluation
             | N6003  | shop  | -                   |
 
 
-    Scenario: Postcode areas
-        When loading osm data
-            """
-            n1 x12.36853 y51.50618
-            n2 x12.36853 y51.42362
-            n3 x12.63666 y51.42362
-            n4 x12.63666 y51.50618
-            w1 Tboundary=postal_code,ref=3456 Nn1,n2,n3,n4,n1
-            """
-        Then place contains exactly
-            | object | class    | type        | name          |
-            | W1     | boundary | postal_code | 'ref': '3456' |
-
     Scenario: Main with extra
         When loading osm data
             """
@@ -135,10 +122,10 @@ Feature: Tag evaluation
             n8003 Tshop=shoes,name:source=survey
             """
         Then place contains exactly
-            | object | class | name | extratags    |
-            | N8001  | shop  |  -   | 'xx': 'yy'   |
-            | N8002  | shop  |  -   | 'ele': '234' |
-            | N8003  | shop  |  -   | -            |
+            | object | class | extratags    |
+            | N8001  | shop  | 'xx': 'yy'   |
+            | N8002  | shop  | 'ele': '234' |
+            | N8003  | shop  | -            |
 
 
     Scenario: Admin levels

test/php/Nominatim/DebugTest.php

@@ -37,14 +37,14 @@ class DebugTest extends \PHPUnit\Framework\TestCase
 <pre><b>Var1:</b>  <i>True</i></pre>
 <pre><b>Var2:</b>  <i>False</i></pre>
 <pre><b>Var3:</b>  0</pre>
-<pre><b>Var4:</b>  &#039;String&#039;</pre>
-<pre><b>Var5:</b>  0 => &#039;one&#039;
-       1 => &#039;two&#039;
-       2 => &#039;three&#039;</pre>
-<pre><b>Var6:</b>  &#039;key&#039; => &#039;value&#039;
-       &#039;key2&#039; => &#039;value2&#039;</pre>
+<pre><b>Var4:</b>  'String'</pre>
+<pre><b>Var5:</b>  0 => 'one'
+       1 => 'two'
+       2 => 'three'</pre>
+<pre><b>Var6:</b>  'key' => 'value'
+       'key2' => 'value2'</pre>
 <pre><b>Var7:</b>  me as string</pre>
-<pre><b>Var8:</b>  &#039;value&#039;, &#039;value2&#039;</pre>
+<pre><b>Var8:</b>  'value', 'value2'</pre>
 
 EOT
         );
@@ -64,10 +64,10 @@ EOT
     public function testDebugArray()
     {
         $this->expectOutputString(<<<EOT
-<pre><b>Arr0:</b>  &#039;null&#039;</pre>
-<pre><b>Arr1:</b>  &#039;key1&#039; => &#039;val1&#039;
-       &#039;key2&#039; => &#039;val2&#039;
-       &#039;key3&#039; => &#039;val3&#039;</pre>
+<pre><b>Arr0:</b>  'null'</pre>
+<pre><b>Arr1:</b>  'key1' => 'val1'
+       'key2' => 'val2'
+       'key3' => 'val3'</pre>
 
 EOT
         );
@@ -93,12 +93,12 @@ EOT
     <th><small>1</small></th>
   </tr>
   <tr>
-    <td><pre>&#039;one&#039;</pre></td>
-    <td><pre>&#039;two&#039;</pre></td>
+    <td><pre>'one'</pre></td>
+    <td><pre>'two'</pre></td>
   </tr>
   <tr>
-    <td><pre>&#039;three&#039;</pre></td>
-    <td><pre>&#039;four&#039;</pre></td>
+    <td><pre>'three'</pre></td>
+    <td><pre>'four'</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -109,9 +109,9 @@ EOT
     <th><small>key3</small></th>
   </tr>
   <tr>
-    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>&#039;val3&#039;</pre></td>
+    <td><pre>'val1'</pre></td>
+    <td><pre>'val2'</pre></td>
+    <td><pre>'val3'</pre></td>
   </tr>
 </table>
 
@@ -147,18 +147,18 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>&#039;val2&#039;</pre></td>
+    <td><pre>'val1'</pre></td>
+    <td><pre>'val2'</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>&#039;one&#039;</pre></td>
-    <td><pre>&#039;two&#039;</pre></td>
+    <td><pre>'one'</pre></td>
+    <td><pre>'two'</pre></td>
   </tr>
   <tr>
     <td><pre>group2</pre></td>
-    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>&#039;val2&#039;</pre></td>
+    <td><pre>'val1'</pre></td>
+    <td><pre>'val2'</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -171,15 +171,15 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>&#039;val3&#039;</pre></td>
+    <td><pre>'val1'</pre></td>
+    <td><pre>'val2'</pre></td>
+    <td><pre>'val3'</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>&#039;val3&#039;</pre></td>
+    <td><pre>'val1'</pre></td>
+    <td><pre>'val2'</pre></td>
+    <td><pre>'val3'</pre></td>
   </tr>
 </table>
 

test/python/tools/test_add_osm_data.py

@@ -24,14 +24,10 @@ class CaptureGetUrl:
         return '<xml></xml>'
 
 
-@pytest.fixture(autouse=True)
-def setup_delete_postprocessing(temp_db_cursor):
-    temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION flush_deleted_places()
-                              RETURNS INTEGER AS $$ SELECT 1 $$ LANGUAGE SQL""")
+def test_import_osm_file_simple(table_factory, osm2pgsql_options, capfd):
+    table_factory('place', content=((1, ), ))
 
-def test_import_osm_file_simple(dsn, table_factory, osm2pgsql_options, capfd):
-
-    assert add_osm_data.add_data_from_file(dsn, Path('change.osm'), osm2pgsql_options) == 0
+    assert add_osm_data.add_data_from_file(Path('change.osm'), osm2pgsql_options) == 0
     captured = capfd.readouterr()
 
     assert '--append' in captured.out
@@ -45,11 +41,11 @@ def test_import_osm_file_simple(dsn, table_factory, osm2pgsql_options, capfd):
 @pytest.mark.parametrize("osm_type", ['node', 'way', 'relation'])
 @pytest.mark.parametrize("main_api,url", [(True, 'https://www.openstreetmap.org/api'),
                                           (False, 'https://overpass-api.de/api/interpreter?')])
-def test_import_osm_object_main_api(dsn, osm2pgsql_options, monkeypatch,
-                                    capfd, osm_type, main_api, url):
+def test_import_osm_object_main_api(osm2pgsql_options, monkeypatch, capfd,
+                                    osm_type, main_api, url):
     get_url_mock = CaptureGetUrl(monkeypatch)
 
-    add_osm_data.add_osm_object(dsn, osm_type, 4536, main_api, osm2pgsql_options)
+    add_osm_data.add_osm_object(osm_type, 4536, main_api, osm2pgsql_options)
     captured = capfd.readouterr()
 
     assert get_url_mock.url.startswith(url)

test/python/tools/test_refresh.py

@@ -48,7 +48,7 @@ def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, r
 def test_recompute_importance(placex_table, table_factory, temp_db_conn, temp_db_cursor):
     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE,
                                               country_code varchar(2),
-                                              rank_search SMALLINT,
+                                              osm_type varchar(1), osm_id BIGINT,
                                               centroid GEOMETRY,
                                               OUT importance FLOAT,
                                               OUT wikipedia TEXT)

vagrant/Install-on-Ubuntu-18.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-10-postgis-2.4 \
                         postgresql-contrib-10 postgresql-10-postgis-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-pip \
-                        python3-psutil python3-jinja2 python3-yaml python3-icu git
+                        python3-psutil python3-jinja2 python3-yaml python3-icu
 
 # Some of the Python packages that come with Ubuntu 18.04 are too old, so
 # install the latest version from pip:
@@ -105,25 +105,18 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    git clone --recursive https://github.com/openstreetmap/Nominatim.git
-    cd Nominatim
+    wget https://nominatim.org/release/Nominatim-4.2.4.tar.bz2
+    tar xf Nominatim-4.2.4.tar.bz2
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
-# When installing the latest source from github, you also need to
-# download the country grid:
-
-if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
-    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
-fi                                 #DOCS:
-
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim
+    cmake $USERHOME/Nominatim-4.2.4
     make
     sudo make install
 

vagrant/Install-on-Ubuntu-20.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-contrib-12 postgresql-12-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \
                         python3-psycopg2 python3-psutil python3-jinja2 \
-                        python3-icu python3-datrie python3-yaml git
+                        python3-icu python3-datrie python3-yaml
 
 #
 # System Configuration
@@ -99,25 +99,18 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    git clone --recursive https://github.com/openstreetmap/Nominatim.git
-    cd Nominatim
+    wget https://nominatim.org/release/Nominatim-4.2.4.tar.bz2
+    tar xf Nominatim-4.2.4.tar.bz2
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
-# When installing the latest source from github, you also need to
-# download the country grid:
-
-if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
-    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
-fi                                 #DOCS:
-
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim
+    cmake $USERHOME/Nominatim-4.2.4
     make
     sudo make install
 

vagrant/Install-on-Ubuntu-22.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-contrib-14 postgresql-14-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \
                         python3-psycopg2 python3-psutil python3-jinja2 \
-                        python3-icu python3-datrie git
+                        python3-icu python3-datrie
 
 #
 # System Configuration
@@ -99,25 +99,18 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    git clone --recursive https://github.com/openstreetmap/Nominatim.git
-    cd Nominatim
+    wget https://nominatim.org/release/Nominatim-4.2.4.tar.bz2
+    tar xf Nominatim-4.2.4.tar.bz2
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
-# When installing the latest source from github, you also need to
-# download the country grid:
-
-if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
-    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
-fi                                 #DOCS:
-
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim
+    cmake $USERHOME/Nominatim-4.2.4
     make
     sudo make install