prepare release 4.2.4

Before postgis 3.4 ST_Project required a geography as input and seemed
to have implicitly converted to geography. Since 3.4 geometry input
is supported but leads to a completely different result.

.github/actions/setup-postgresql/action.yml

@@ -15,7 +15,9 @@ runs:
         - name: Remove existing PostgreSQL
           run: |
               sudo apt-get purge -yq postgresql*
-              sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+              sudo apt install curl ca-certificates gnupg
+              curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
+              sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
               sudo apt-get update -qq
 
           shell: bash

.github/workflows/ci-tests.yml

@@ -37,13 +37,8 @@ jobs:
         needs: create-archive
         strategy:
             matrix:
-                ubuntu: [18, 20, 22]
+                ubuntu: [20, 22]
                 include:
-                    - ubuntu: 18
-                      postgresql: 9.6
-                      postgis: 2.5
-                      pytest: pytest
-                      php: 7.2
                     - ubuntu: 20
                       postgresql: 13
                       postgis: 3
@@ -69,8 +64,10 @@ jobs:
               uses: shivammathur/setup-php@v2
               with:
                   php-version: ${{ matrix.php }}
-                  tools: phpunit, phpcs, composer
+                  tools: phpunit:9, phpcs, composer
                   ini-values: opcache.jit=disable
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
             - uses: actions/setup-python@v4
               with:
@@ -100,18 +97,22 @@ jobs:
 
             - name: Install latest pylint/mypy
               run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions
+              if: matrix.ubuntu == 22
 
             - name: PHP linting
               run: phpcs --report-width=120 .
               working-directory: Nominatim
+              if: matrix.ubuntu == 22
 
             - name: Python linting
               run: pylint nominatim
               working-directory: Nominatim
+              if: matrix.ubuntu == 22
 
             - name: Python static typechecking
               run: mypy --strict nominatim
               working-directory: Nominatim
+              if: matrix.ubuntu == 22
 
 
             - name: PHP unit tests

.pylintrc

@@ -13,6 +13,6 @@ ignored-classes=NominatimArgs,closing
 # 'too-many-ancestors' is triggered already by deriving from UserDict
 # 'not-context-manager' disabled because it causes false positives once
 #   typed Python is enabled. See also https://github.com/PyCQA/pylint/issues/5273
-disable=too-few-public-methods,duplicate-code,too-many-ancestors,bad-option-value,no-self-use,not-context-manager
+disable=too-few-public-methods,duplicate-code,too-many-ancestors,bad-option-value,no-self-use,not-context-manager,use-dict-literal
 
 good-names=i,x,y,m,fd,db,cc

CMakeLists.txt

@@ -20,7 +20,7 @@ project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 4)
 set(NOMINATIM_VERSION_MINOR 2)
-set(NOMINATIM_VERSION_PATCH 0)
+set(NOMINATIM_VERSION_PATCH 4)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 
@@ -269,6 +269,12 @@ install(FILES settings/env.defaults
               settings/import-address.style
               settings/import-full.style
               settings/import-extratags.style
+              settings/import-admin.lua
+              settings/import-street.lua
+              settings/import-address.lua
+              settings/import-full.lua
+              settings/import-extratags.lua
+              settings/flex-base.lua
               settings/icu_tokenizer.yaml
               settings/country_settings.yaml
         DESTINATION ${NOMINATIM_CONFIGDIR})

ChangeLog

@@ -1,3 +1,33 @@
+4.2.4
+ * fix a potential SQL injection in 'nominatim admin --collect-os-info'
+ * fix compatibility issue with PostGIS 3.4
+
+4.2.3
+
+ * fix deletion handling for 'nominatim add-data'
+ * adapt place_force_delete() to new deletion handling
+ * flex style: avoid dropping of postcode areas
+ * fix update errors on address interpolation handling
+
+4.2.2
+
+ * extend flex-style library to fully support all default styles
+ * fix handling of Hebrew aleph
+ * do not assign postcodes to rivers
+ * fix string matching in PHP code
+ * update osm2pgsql (various updates to flex)
+ * fix slow query when deleting places on update
+ * fix CLI details query
+ * fix recalculation of importance values
+ * fix polygon simplification in reverse results
+ * add class/type information to reverse geocodejson result
+ * minor improvements to default tokenizer configuration
+ * various smaller fixes to documentation
+
+4.2.1
+
+ * fix XSS vulnerability in debug view
+
 4.2.0
 
  * add experimental support for osm2pgsql flex style
@@ -21,6 +51,10 @@
  * typing fixes to work with latest type annotations from typeshed
  * smaller improvements to documentation (thanks to @mausch)
 
+4.1.1
+
+ * fix XSS vulnerability in debug view
+
 4.1.0
 
  * switch to ICU tokenizer as default
@@ -57,6 +91,10 @@
  * add setup instructions for updates and systemd
  * drop support for PostgreSQL 9.5
 
+4.0.2
+
+ * fix XSS vulnerability in debug view
+
 4.0.1
 
  * fix initialisation error in replication script
@@ -95,6 +133,10 @@
  * add testing of installation scripts via CI
  * drop support for Python < 3.6 and Postgresql < 9.5
 
+3.7.3
+
+ * fix XSS vulnerability in debug view
+
 3.7.2
 
  * fix database check for reverse-only imports

VAGRANT.md

@@ -1,6 +1,6 @@
 # Install Nominatim in a virtual machine for development and testing
 
-This document describes how you can install Nominatim inside a Ubuntu 16
+This document describes how you can install Nominatim inside a Ubuntu 22
 virtual machine on your desktop/laptop (host machine). The goal is to give
 you a development environment to easily edit code and run the test suite
 without affecting the rest of your system. 
@@ -69,8 +69,7 @@ installation.
 PHP errors are written to `/var/log/apache2/error.log`.
 
 With `echo` and `var_dump()` you write into the output (HTML/XML/JSON) when
-you either add `&debug=1` to the URL (preferred) or set
+you either add `&debug=1` to the URL.
-`@define('CONST_Debug', true);` in `settings/local.php`.
 
 In the Python BDD test you can use `logger.info()` for temporary debug
 statements.
@@ -130,6 +129,10 @@ and then
 Yes, Vagrant and Virtualbox can be installed on MS Windows just fine. You need a 64bit
 version of Windows.
 
+##### Will it run on Apple Silicon?
+
+You might need to replace Virtualbox with [Parallels](https://www.parallels.com/products/desktop/).
+There is no free/open source version of Parallels.
 
 ##### Why Monaco, can I use another country?
 
@@ -141,11 +144,12 @@ No. Long running Nominatim installations will differ once new import features (o
 bug fixes) get added since those usually only get applied to new/changed data.
 
 Also this document skips the optional Wikipedia data import which affects ranking
-of search results. See [Nominatim installation](https://nominatim.org/release-docs/latest/admin/Installation) for details.
+of search results. See [Nominatim installation](https://nominatim.org/release-docs/latest/admin/Installation)
+for details.
 
 ##### Why Ubuntu? Can I test CentOS/Fedora/CoreOS/FreeBSD?
 
-There is a Vagrant script for CentOS available, but the Nominatim directory
+There used to be a Vagrant script for CentOS available, but the Nominatim directory
 isn't symlinked/mounted to the host which makes development trickier. We used
 it mainly for debugging installation with SELinux.
 
@@ -154,14 +158,17 @@ are slightly different, e.g. the name of the package manager, Apache2 package
 name, location of files. We chose Ubuntu because that is closest to the
 nominatim.openstreetmap.org production environment.
 
-You can configure/download other Vagrant boxes from [https://app.vagrantup.com/boxes/search](https://app.vagrantup.com/boxes/search).
+You can configure/download other Vagrant boxes from
+[https://app.vagrantup.com/boxes/search](https://app.vagrantup.com/boxes/search).
 
 ##### How can I connect to an existing database?
 
-Let's say you have a Postgres database named `nominatim_it` on server `your-server.com` and port `5432`. The Postgres username is `postgres`. You can edit `settings/local.php` and point Nominatim to it.
+Let's say you have a Postgres database named `nominatim_it` on server `your-server.com`
+and port `5432`. The Postgres username is `postgres`. You can edit the `.env` in your
+project directory and point Nominatim to it.
+
+    NOMINATIM_DATABASE_DSN="pgsql:host=your-server.com;port=5432;user=postgres;dbname=nominatim_it
 
-    pgsql:host=your-server.com;port=5432;user=postgres;dbname=nominatim_it
-    
 No data import or restarting necessary.
 
 If the Postgres installation is behind a firewall, you can try
@@ -169,11 +176,12 @@ If the Postgres installation is behind a firewall, you can try
     ssh -L 9999:localhost:5432 your-username@your-server.com
 
 inside the virtual machine. It will map the port to `localhost:9999` and then
-you edit `settings/local.php` with
+you edit `.env` file with
 
-    @define('CONST_Database_DSN', 'pgsql:host=localhost;port=9999;user=postgres;dbname=nominatim_it');
+    NOMINATIM_DATABASE_DSN="pgsql:host=localhost;port=9999;user=postgres;dbname=nominatim_it"
 
-To access postgres directly remember to specify the hostname, e.g. `psql --host localhost --port 9999 nominatim_it`
+To access postgres directly remember to specify the hostname,
+e.g. `psql --host localhost --port 9999 nominatim_it`
 
 
 ##### My computer is slow and the import takes too long. Can I start the virtual machine "in the cloud"?

docs/admin/Migration.md

@@ -15,14 +15,6 @@ breaking changes. **Please read them before running the migration.**
     If you are migrating from a version <3.6, then you still have to follow
     the manual migration steps up to 3.6.
 
-## 4.2.2 -> 4.2.3
-
-### Update interpolation functions
-
-When updating to this release, you need to run `nominatim refresh --functions`
-after updating and before restarting updates. Otherwise you may see an error
-`Splitting of Point geometries is unsupported` or similar.
-
 ## 4.0.0 -> 4.1.0
 
 ### ICU tokenizer is the new default

docs/extra.css

@@ -1,3 +1,7 @@
+.toctree-l3 {
+    display: none!important
+}
+
 table {
     margin-bottom: 12pt
 }

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Nominatim 4.2.4
+site_name: Nominatim Documentation
 theme: readthedocs
 docs_dir: ${CMAKE_CURRENT_BINARY_DIR}
 site_url: https://nominatim.org

lib-php/DebugHtml.php

@@ -135,7 +135,7 @@ class Debug
 
     public static function printSQL($sSQL)
     {
-        echo '<p><tt><font color="#aaa">'.$sSQL.'</font></tt></p>'."\n";
+        echo '<p><tt><font color="#aaa">'.htmlspecialchars($sSQL, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401).'</font></tt></p>'."\n";
     }
 
     private static function outputVar($mVar, $sPreNL)
@@ -178,11 +178,12 @@ class Debug
         }
 
         if (is_string($mVar)) {
-            echo "'$mVar'";
+            $sOut = "'$mVar'";
-            return strlen($mVar) + 2;
+        } else {
+            $sOut = (string)$mVar;
         }
 
-        echo (string)$mVar;
+        echo htmlspecialchars($sOut, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401);
-        return strlen((string)$mVar);
+        return strlen($sOut);
     }
 }

lib-php/Geocode.php

@@ -874,7 +874,7 @@ class Geocode
                 $iCountWords = 0;
                 $sAddress = $aResult['langaddress'];
                 foreach ($aRecheckWords as $i => $sWord) {
-                    if (stripos($sAddress, $sWord)!==false) {
+                    if (grapheme_stripos($sAddress, $sWord)!==false) {
                         $iCountWords++;
                         if (preg_match('/(^|,)\s*'.preg_quote($sWord, '/').'\s*(,|$)/', $sAddress)) {
                             $iCountWords += 0.1;

lib-php/PlaceLookup.php

@@ -524,12 +524,7 @@ class PlaceLookup
 
         // Get the bounding box and outline polygon
         $sSQL = 'select place_id,0 as numfeatures,st_area(geometry) as area,';
-        if ($fLonReverse != null && $fLatReverse != null) {
+        $sSQL .= ' ST_Y(centroid) as centrelat, ST_X(centroid) as centrelon,';
-            $sSQL .= ' ST_Y(closest_point) as centrelat,';
-            $sSQL .= ' ST_X(closest_point) as centrelon,';
-        } else {
-            $sSQL .= ' ST_Y(centroid) as centrelat, ST_X(centroid) as centrelon,';
-        }
         $sSQL .= ' ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,';
         $sSQL .= ' ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon';
         if ($this->bIncludePolygonAsGeoJSON) {
@@ -544,19 +539,21 @@ class PlaceLookup
         if ($this->bIncludePolygonAsText) {
             $sSQL .= ',ST_AsText(geometry) as astext';
         }
+
+        $sSQL .= ' FROM (SELECT place_id';
         if ($fLonReverse != null && $fLatReverse != null) {
-            $sFrom = ' from (SELECT * , CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
+            $sSQL .= ',CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
-            $sFrom .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
+            $sSQL .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
-            $sFrom .=' ELSE centroid END AS closest_point';
+            $sSQL .=' ELSE centroid END AS centroid';
-            $sFrom .= ' from placex where place_id = '.$iPlaceID.') as plx';
         } else {
-            $sFrom = ' from placex where place_id = '.$iPlaceID;
+            $sSQL .= ',centroid';
         }
         if ($this->fPolygonSimplificationThreshold > 0) {
-            $sSQL .= ' from (select place_id,centroid,ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry'.$sFrom.') as plx';
+            $sSQL .= ',ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry';
         } else {
-            $sSQL .= $sFrom;
+            $sSQL .= ',geometry';
         }
+        $sSQL .= ' FROM placex where place_id = '.$iPlaceID.') as plx';
 
         $aPointPolygon = $this->oDB->getRow($sSQL, null, 'Could not get outline');
 

lib-php/template/address-geocodejson.php

@@ -36,6 +36,9 @@ if (empty($aPlace)) {
         $aFilteredPlaces['properties']['geocoding']['osm_id'] = $aPlace['osm_id'];
     }
 
+    $aFilteredPlaces['properties']['geocoding']['osm_key'] = $aPlace['class'];
+    $aFilteredPlaces['properties']['geocoding']['osm_value'] = $aPlace['type'];
+
     $aFilteredPlaces['properties']['geocoding']['type'] = addressRankToGeocodeJsonType($aPlace['rank_address']);
 
     $aFilteredPlaces['properties']['geocoding']['accuracy'] = (int) $fDistance;

lib-sql/functions/interpolation.sql

@@ -164,7 +164,7 @@ DECLARE
   newend INTEGER;
   moddiff SMALLINT;
   linegeo GEOMETRY;
-  splitline GEOMETRY;
+  splitpoint FLOAT;
   sectiongeo GEOMETRY;
   postcode TEXT;
   stepmod SMALLINT;
@@ -223,15 +223,27 @@ BEGIN
         FROM placex, generate_series(1, array_upper(waynodes, 1)) nodeidpos
         WHERE osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT
               and address is not NULL and address ? 'housenumber'
+              and ST_Distance(NEW.linegeo, geometry) < 0.0005
         ORDER BY nodeidpos
     LOOP
       {% if debug %}RAISE WARNING 'processing point % (%)', nextnode.hnr, ST_AsText(nextnode.geometry);{% endif %}
       IF linegeo is null THEN
         linegeo := NEW.linegeo;
       ELSE
-        splitline := ST_Split(ST_Snap(linegeo, nextnode.geometry, 0.0005), nextnode.geometry);
+        splitpoint := ST_LineLocatePoint(linegeo, nextnode.geometry);
-        sectiongeo := ST_GeometryN(splitline, 1);
+        IF splitpoint = 0 THEN
-        linegeo := ST_GeometryN(splitline, 2);
+          -- Corner case where the splitpoint falls on the first point
+          -- and thus would not return a geometry. Skip that section.
+          sectiongeo := NULL;
+        ELSEIF splitpoint = 1 THEN
+          -- Point is at the end of the line.
+          sectiongeo := linegeo;
+          linegeo := NULL;
+        ELSE
+          -- Split the line.
+          sectiongeo := ST_LineSubstring(linegeo, 0, splitpoint);
+          linegeo := ST_LineSubstring(linegeo, splitpoint, 1);
+        END IF;
       END IF;
 
       IF prevnode.hnr is not null
@@ -239,6 +251,9 @@ BEGIN
          -- regularly mapped housenumbers.
          -- (Conveniently also fails if one of the house numbers is not a number.)
          and abs(prevnode.hnr - nextnode.hnr) > NEW.step
+         -- If the interpolation geometry is broken or two nodes are at the
+         -- same place, then splitting might produce a point. Ignore that.
+         and ST_GeometryType(sectiongeo) = 'ST_LineString'
       THEN
         IF prevnode.hnr < nextnode.hnr THEN
           startnumber := prevnode.hnr;
@@ -300,12 +315,12 @@ BEGIN
                   NEW.address, postcode,
                   NEW.country_code, NEW.geometry_sector, 0);
         END IF;
+      END IF;
 
-        -- early break if we are out of line string,
+      -- early break if we are out of line string,
-        -- might happen when a line string loops back on itself
+      -- might happen when a line string loops back on itself
-        IF ST_GeometryType(linegeo) != 'ST_LineString' THEN
+      IF linegeo is null or ST_GeometryType(linegeo) != 'ST_LineString' THEN
-            RETURN NEW;
+          RETURN NEW;
-        END IF;
       END IF;
 
       prevnode := nextnode;

lib-sql/functions/place_triggers.sql

@@ -384,7 +384,19 @@ BEGIN
 
   -- Mark for delete in the placex table
   UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
-    WHERE placex.osm_type = place_to_be_deleted.osm_type
+    WHERE placex.osm_type = 'N' and place_to_be_deleted.osm_type = 'N'
+          and placex.osm_id = place_to_be_deleted.osm_id
+          and placex.class = place_to_be_deleted.class
+          and placex.type = place_to_be_deleted.type
+          and not deferred;
+  UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
+    WHERE placex.osm_type = 'W' and place_to_be_deleted.osm_type = 'W'
+          and placex.osm_id = place_to_be_deleted.osm_id
+          and placex.class = place_to_be_deleted.class
+          and placex.type = place_to_be_deleted.type
+          and not deferred;
+  UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
+    WHERE placex.osm_type = 'R' and place_to_be_deleted.osm_type = 'R'
           and placex.osm_id = place_to_be_deleted.osm_id
           and placex.class = place_to_be_deleted.class
           and placex.type = place_to_be_deleted.type

lib-sql/functions/placex_triggers.sql

@@ -1120,7 +1120,7 @@ BEGIN
   ELSE
     -- No linked place? As a last resort check if the boundary is tagged with
     -- a place type and adapt the rank address.
-    IF NEW.rank_address > 0 and NEW.extratags ? 'place' THEN
+    IF NEW.rank_address between 4 and 25 and NEW.extratags ? 'place' THEN
       SELECT address_rank INTO place_address_level
         FROM compute_place_rank(NEW.country_code, 'A', 'place',
                                 NEW.extratags->'place', 0::SMALLINT, False, null);
@@ -1230,7 +1230,11 @@ BEGIN
     {% endif %}
   END IF;
 
-  IF NEW.postcode is null AND NEW.rank_search > 8 THEN
+  IF NEW.postcode is null AND NEW.rank_search > 8
+     AND (NEW.rank_address > 0
+          OR ST_GeometryType(NEW.geometry) not in ('ST_LineString','ST_MultiLineString')
+          OR ST_Length(NEW.geometry) < 0.02)
+  THEN
     NEW.postcode := get_nearest_postcode(NEW.country_code, NEW.geometry);
   END IF;
 

lib-sql/functions/utils.sql

@@ -273,8 +273,8 @@ BEGIN
   END IF;
 
   RETURN ST_Envelope(ST_Collect(
-                     ST_Project(geom, radius, 0.785398)::geometry,
+                     ST_Project(geom::geography, radius, 0.785398)::geometry,
-                     ST_Project(geom, radius, 3.9269908)::geometry));
+                     ST_Project(geom::geography, radius, 3.9269908)::geometry));
 END;
 $$
 LANGUAGE plpgsql IMMUTABLE;
@@ -429,9 +429,10 @@ BEGIN
   SELECT osm_type, osm_id, class, type FROM placex WHERE place_id = placeid INTO osmtype, osmid, pclass, ptype;
   DELETE FROM import_polygon_delete where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
   DELETE FROM import_polygon_error where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
-  -- force delete from place/placex by making it a very small geometry
+  -- force delete by directly entering it into the to-be-deleted table
-  UPDATE place set geometry = ST_SetSRID(ST_Point(0,0), 4326) where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
+  INSERT INTO place_to_be_deleted (osm_type, osm_id, class, type, deferred)
-  DELETE FROM place where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
+         VALUES(osmtype, osmid, pclass, ptype, false);
+  PERFORM flush_deleted_places();
 
   RETURN TRUE;
 END;

nominatim/clicmd/add_data.py

@@ -76,21 +76,25 @@ class UpdateAddData:
 
         osm2pgsql_params = args.osm2pgsql_options(default_cache=1000, default_threads=1)
         if args.file or args.diff:
-            return add_osm_data.add_data_from_file(cast(str, args.file or args.diff),
+            return add_osm_data.add_data_from_file(args.config.get_libpq_dsn(),
+                                                   cast(str, args.file or args.diff),
                                                    osm2pgsql_params)
 
         if args.node:
-            return add_osm_data.add_osm_object('node', args.node,
+            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
+                                               'node', args.node,
                                                args.use_main_api,
                                                osm2pgsql_params)
 
         if args.way:
-            return add_osm_data.add_osm_object('way', args.way,
+            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
+                                               'way', args.way,
                                                args.use_main_api,
                                                osm2pgsql_params)
 
         if args.relation:
-            return add_osm_data.add_osm_object('relation', args.relation,
+            return add_osm_data.add_osm_object(args.config.get_libpq_dsn(),
+                                               'relation', args.relation,
                                                args.use_main_api,
                                                osm2pgsql_params)
 

nominatim/clicmd/api.py

@@ -248,9 +248,9 @@ class APIDetails:
         if args.node:
             params = dict(osmtype='N', osmid=args.node)
         elif args.way:
-            params = dict(osmtype='W', osmid=args.node)
+            params = dict(osmtype='W', osmid=args.way)
         elif args.relation:
-            params = dict(osmtype='R', osmid=args.node)
+            params = dict(osmtype='R', osmid=args.relation)
         else:
             params = dict(place_id=args.place_id)
         if args.object_class:

nominatim/db/async_connection.py

@@ -69,8 +69,8 @@ class DBConnection:
         self.current_params: Optional[Sequence[Any]] = None
         self.ignore_sql_errors = ignore_sql_errors
 
-        self.conn: Optional['psycopg2.connection'] = None
+        self.conn: Optional['psycopg2._psycopg.connection'] = None
-        self.cursor: Optional['psycopg2.cursor'] = None
+        self.cursor: Optional['psycopg2._psycopg.cursor'] = None
         self.connect(cursor_factory=cursor_factory)
 
     def close(self) -> None:
@@ -78,7 +78,7 @@ class DBConnection:
         """
         if self.conn is not None:
             if self.cursor is not None:
-                self.cursor.close() # type: ignore[no-untyped-call]
+                self.cursor.close()
                 self.cursor = None
             self.conn.close()
 

nominatim/db/connection.py

@@ -31,7 +31,7 @@ class Cursor(psycopg2.extras.DictCursor):
         """ Query execution that logs the SQL query when debugging is enabled.
         """
         if LOG.isEnabledFor(logging.DEBUG):
-            LOG.debug(self.mogrify(query, args).decode('utf-8')) # type: ignore[no-untyped-call]
+            LOG.debug(self.mogrify(query, args).decode('utf-8'))
 
         super().execute(query, args)
 

nominatim/db/utils.py

@@ -118,4 +118,4 @@ class CopyBuffer:
         """
         if self.buffer.tell() > 0:
             self.buffer.seek(0)
-            cur.copy_from(self.buffer, table, columns=columns) # type: ignore[no-untyped-call]
+            cur.copy_from(self.buffer, table, columns=columns)

nominatim/tools/add_osm_data.py

@@ -12,23 +12,34 @@ from pathlib import Path
 import logging
 import urllib
 
+from nominatim.db.connection import connect
 from nominatim.tools.exec_utils import run_osm2pgsql, get_url
 
 LOG = logging.getLogger()
 
-def add_data_from_file(fname: str, options: MutableMapping[str, Any]) -> int:
+def _run_osm2pgsql(dsn: str, options: MutableMapping[str, Any]) -> None:
+    run_osm2pgsql(options)
+
+    # Handle deletions
+    with connect(dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute('SELECT flush_deleted_places()')
+        conn.commit()
+
+
+def add_data_from_file(dsn: str, fname: str, options: MutableMapping[str, Any]) -> int:
     """ Adds data from a OSM file to the database. The file may be a normal
         OSM file or a diff file in all formats supported by libosmium.
     """
     options['import_file'] = Path(fname)
     options['append'] = True
-    run_osm2pgsql(options)
+    _run_osm2pgsql(dsn, options)
 
     # No status update. We don't know where the file came from.
     return 0
 
 
-def add_osm_object(osm_type: str, osm_id: int, use_main_api: bool,
+def add_osm_object(dsn: str, osm_type: str, osm_id: int, use_main_api: bool,
                    options: MutableMapping[str, Any]) -> int:
     """ Add or update a single OSM object from the latest version of the
         API.
@@ -51,6 +62,6 @@ def add_osm_object(osm_type: str, osm_id: int, use_main_api: bool,
     options['append'] = True
     options['import_data'] = get_url(base_url).encode('utf-8')
 
-    run_osm2pgsql(options)
+    _run_osm2pgsql(dsn, options)
 
     return 0

nominatim/tools/collect_os_info.py

@@ -12,14 +12,13 @@ import os
 import subprocess
 import sys
 from pathlib import Path
-from typing import List, Optional, Tuple, Union, cast
+from typing import List, Optional, Tuple, Union
 
 import psutil
 from psycopg2.extensions import make_dsn, parse_dsn
 
 from nominatim.config import Configuration
 from nominatim.db.connection import connect
-from nominatim.typing import DictCursorResults
 from nominatim.version import version_str
 
 
@@ -107,15 +106,15 @@ def report_system_information(config: Configuration) -> None:
         postgresql_ver: str = convert_version(conn.server_version_tuple())
 
         with conn.cursor() as cur:
-            cur.execute(f"""
+            num = cur.scalar("SELECT count(*) FROM pg_catalog.pg_database WHERE datname=%s",
-            SELECT datname FROM pg_catalog.pg_database 
+                             (parse_dsn(config.get_libpq_dsn())['dbname'], ))
-            WHERE datname='{parse_dsn(config.get_libpq_dsn())['dbname']}'""")
+            nominatim_db_exists = num == 1 if isinstance(num, int) else False
-            nominatim_db_exists = cast(Optional[DictCursorResults], cur.fetchall())
+
-            if nominatim_db_exists:
+    if nominatim_db_exists:
-                with connect(config.get_libpq_dsn()) as conn:
+        with connect(config.get_libpq_dsn()) as conn:
-                    postgis_ver: str = convert_version(conn.postgis_version_tuple())
+            postgis_ver: str = convert_version(conn.postgis_version_tuple())
-            else:
+    else:
-                postgis_ver = "Unable to connect to database"
+        postgis_ver = "Unable to connect to database"
 
     postgresql_config: str = get_postgresql_config(int(float(postgresql_ver)))
 

nominatim/tools/exec_utils.py

@@ -118,7 +118,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
     cmd = [str(options['osm2pgsql']),
            '--hstore', '--latlon', '--slim',
            '--log-progress', 'true',
-           '--number-processes', str(options['threads']),
+           '--number-processes', '1' if options['append'] else str(options['threads']),
            '--cache', str(options['osm2pgsql_cache']),
            '--style', str(options['osm2pgsql_style'])
           ]

nominatim/tools/refresh.py

@@ -176,7 +176,7 @@ def recompute_importance(conn: Connection) -> None:
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance
-                FROM compute_importance(extratags, country_code, osm_type, osm_id, centroid))
+                FROM compute_importance(extratags, country_code, rank_search, centroid))
             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance

nominatim/version.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2022 by the Nominatim developer community.
+# Copyright (C) 2023 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Version information for Nominatim.
@@ -25,7 +25,7 @@ from typing import Optional, Tuple
 # patch level when cherry-picking the commit with the migration.
 #
 # Released versions always have a database patch level of 0.
-NOMINATIM_VERSION = (4, 2, 0, 0)
+NOMINATIM_VERSION = (4, 2, 4, 0)
 
 POSTGRESQL_REQUIRED_VERSION = (9, 6)
 POSTGIS_REQUIRED_VERSION = (2, 2)

settings/flex-base.lua

@@ -1,9 +1,19 @@
 -- Core functions for Nominatim import flex style.
 --
 
+local module = {}
+
+local PRE_DELETE = nil
+local PRE_EXTRAS = nil
+local MAIN_KEYS = nil
+local NAMES = nil
+local ADDRESS_TAGS = nil
+local SAVE_EXTRA_MAINS = false
+local POSTCODE_FALLBACK = true
+
 
 -- The single place table.
-place_table = osm2pgsql.define_table{
+local place_table = osm2pgsql.define_table{
     name = "place",
     ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
     columns = {
@@ -14,7 +24,25 @@ place_table = osm2pgsql.define_table{
         { column = 'address', type = 'hstore' },
         { column = 'extratags', type = 'hstore' },
         { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
-    }
+    },
+    indexes = {}
+}
+
+------------ Geometry functions for relations ---------------------
+
+function module.relation_as_multipolygon(o)
+    return o:as_multipolygon()
+end
+
+function module.relation_as_multiline(o)
+    return o:as_multilinestring():line_merge()
+end
+
+
+module.RELATION_TYPES = {
+    multipolygon = module.relation_as_multipolygon,
+    boundary = module.relation_as_multipolygon,
+    waterway = module.relation_as_multiline
 }
 
 ------------- Place class ------------------------------------------
@@ -43,6 +71,17 @@ function Place.new(object, geom_func)
     return self
 end
 
+function Place:clean(data)
+    for k, v in pairs(self.object.tags) do
+        if data.delete ~= nil and data.delete(k, v) then
+            self.object.tags[k] = nil
+        elseif data.extra ~= nil and data.extra(k, v) then
+            self.extratags[k] = v
+            self.object.tags[k] = nil
+        end
+    end
+end
+
 function Place:delete(data)
     if data.match ~= nil then
         for k, v in pairs(self.object.tags) do
@@ -69,54 +108,37 @@ function Place:grab_extratags(data)
     return count
 end
 
-function Place:grab_address(data)
+local function strip_address_prefix(k)
+    if k:sub(1, 5) == 'addr:' then
+        return k:sub(6)
+    end
+
+    if k:sub(1, 6) == 'is_in:' then
+        return k:sub(7)
+    end
+
+    return k
+end
+
+
+function Place:grab_address_parts(data)
     local count = 0
 
-    if data.match ~= nil then
+    if data.groups ~= nil then
         for k, v in pairs(self.object.tags) do
-            if data.match(k, v) then
+            local atype = data.groups(k, v)
-                self.object.tags[k] = nil
 
-                if data.include_on_name == true then
+            if atype ~= nil then
+                if atype == 'main' then
                     self.has_name = true
-                end
+                    self.address[strip_address_prefix(k)] = v
-
+                    count = count + 1
-                if data.out_key ~= nil then
+                elseif atype == 'extra' then
-                    self.address[data.out_key] = v
+                    self.address[strip_address_prefix(k)] = v
-                    return 1
-                end
-
-                if k:sub(1, 5) == 'addr:' then
-                    self.address[k:sub(6)] = v
-                elseif k:sub(1, 6) == 'is_in:' then
-                    self.address[k:sub(7)] = v
                 else
-                    self.address[k] = v
+                    self.address[atype] = v
                 end
-                count = count + 1
-            end
-        end
-    end
-
-    return count
-end
-
-function Place:set_address(key, value)
-    self.address[key] = value
-end
-
-function Place:grab_name(data)
-    local count = 0
-
-    if data.match ~= nil then
-        for k, v in pairs(self.object.tags) do
-            if data.match(k, v) then
                 self.object.tags[k] = nil
-                self.names[k] = v
-                if data.include_on_name ~= false then
-                    self.has_name = true
-                end
-                count = count + 1
             end
         end
     end
@@ -124,13 +146,30 @@ function Place:grab_name(data)
     return count
 end
 
-function Place:grab_tag(key)
+
-    return self.object:grab_tag(key)
+function Place:grab_name_parts(data)
+    local fallback = nil
+
+    if data.groups ~= nil then
+        for k, v in pairs(self.object.tags) do
+            local atype = data.groups(k, v)
+
+            if atype ~= nil then
+                self.names[k] = v
+                self.object.tags[k] = nil
+                if atype == 'main' then
+                    self.has_name = true
+                elseif atype == 'house' then
+                    self.has_name = true
+                    fallback = {'place', 'house', 'always'}
+                end
+            end
+        end
+    end
+
+    return fallback
 end
 
-function Place:tags()
-    return self.object.tags
-end
 
 function Place:write_place(k, v, mtype, save_extra_mains)
     if mtype == nil then
@@ -184,9 +223,9 @@ function Place:write_row(k, v, save_extra_mains)
         return 0
     end
 
-    if save_extra_mains then
+    if save_extra_mains ~= nil then
         for extra_k, extra_v in pairs(self.object.tags) do
-            if extra_k ~= k then
+            if extra_k ~= k and save_extra_mains(extra_k, extra_v) then
                 self.extratags[extra_k] = extra_v
             end
         end
@@ -204,7 +243,9 @@ function Place:write_row(k, v, save_extra_mains)
 
     if save_extra_mains then
         for k, v in pairs(self.object.tags) do
-            self.extratags[k] = nil
+            if save_extra_mains(k, v) then
+                self.extratags[k] = nil
+            end
         end
     end
 
@@ -214,7 +255,7 @@ function Place:write_row(k, v, save_extra_mains)
 end
 
 
-function tag_match(data)
+function module.tag_match(data)
     if data == nil or next(data) == nil then
         return nil
     end
@@ -276,17 +317,72 @@ function tag_match(data)
 end
 
 
+function module.tag_group(data)
+    if data == nil or next(data) == nil then
+        return nil
+    end
+
+    local fullmatches = {}
+    local key_prefixes = {}
+    local key_suffixes = {}
+
+    for group, tags in pairs(data) do
+        for _, key in pairs(tags) do
+            if key:sub(1, 1) == '*' then
+                if #key > 1 then
+                    if key_suffixes[#key - 1] == nil then
+                        key_suffixes[#key - 1] = {}
+                    end
+                    key_suffixes[#key - 1][key:sub(2)] = group
+                end
+            elseif key:sub(#key, #key) == '*' then
+                if key_prefixes[#key - 1] == nil then
+                    key_prefixes[#key - 1] = {}
+                end
+                key_prefixes[#key - 1][key:sub(1, #key - 1)] = group
+            else
+                fullmatches[key] = group
+            end
+        end
+    end
+
+    return function (k, v)
+        local val = fullmatches[k]
+        if val ~= nil then
+            return val
+        end
+
+        for slen, slist in pairs(key_suffixes) do
+            if #k >= slen then
+                val = slist[k:sub(-slen)]
+                if val ~= nil then
+                    return val
+                end
+            end
+        end
+
+        for slen, slist in pairs(key_prefixes) do
+            if #k >= slen then
+                val = slist[k:sub(1, slen)]
+                if val ~= nil then
+                    return val
+                end
+            end
+        end
+    end
+end
+
 -- Process functions for all data types
-function osm2pgsql.process_node(object)
+function module.process_node(object)
 
     local function geom_func(o)
         return o:as_point()
     end
 
-    process_tags(Place.new(object, geom_func))
+    module.process_tags(Place.new(object, geom_func))
 end
 
-function osm2pgsql.process_way(object)
+function module.process_way(object)
 
     local function geom_func(o)
         local geom = o:as_polygon()
@@ -298,30 +394,24 @@ function osm2pgsql.process_way(object)
         return geom
     end
 
-    process_tags(Place.new(object, geom_func))
+    module.process_tags(Place.new(object, geom_func))
 end
 
-function relation_as_multipolygon(o)
+function module.process_relation(object)
-    return o:as_multipolygon()
+    local geom_func = module.RELATION_TYPES[object.tags.type]
-end
-
-function relation_as_multiline(o)
-    return o:as_multilinestring():line_merge()
-end
-
-function osm2pgsql.process_relation(object)
-    local geom_func = RELATION_TYPES[object.tags.type]
 
     if geom_func ~= nil then
-        process_tags(Place.new(object, geom_func))
+        module.process_tags(Place.new(object, geom_func))
     end
 end
 
-function process_tags(o)
+-- The process functions are used by default by osm2pgsql.
-    local fallback
+osm2pgsql.process_node = module.process_node
+osm2pgsql.process_way = module.process_way
+osm2pgsql.process_relation = module.process_relation
 
-    o:delete{match = PRE_DELETE}
+function module.process_tags(o)
-    o:grab_extratags{match = PRE_EXTRAS}
+    o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS}
 
     -- Exception for boundary/place double tagging
     if o.object.tags.boundary == 'administrative' then
@@ -330,54 +420,91 @@ function process_tags(o)
         end}
     end
 
+    -- name keys
+    local fallback = o:grab_name_parts{groups=NAMES}
+
     -- address keys
-    o:grab_address{match=COUNTRY_TAGS, out_key='country'}
+    if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then
+        fallback = {'place', 'house', 'always'}
+    end
     if o.address.country ~= nil and #o.address.country ~= 2 then
         o.address['country'] = nil
     end
-    if o:grab_name{match=HOUSENAME_TAGS} > 0 then
+    if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then
-        fallback = {'place', 'house'}
+        fallback = {'place', 'postcode', 'always'}
-    end
-    if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then
-        fallback = {'place', 'house'}
-    end
-    if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then
-        fallback = {'place', 'postcode'}
     end
 
-    local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0
+    if o.address.interpolation ~= nil then
-
-    o:grab_address{match=ADDRESS_TAGS}
-
-    if is_interpolation then
         o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
         return
     end
 
-    -- name keys
+    o:clean{delete = POST_DELETE}
-    o:grab_name{match = NAMES}
-    o:grab_name{match = REFS, include_on_name = false}
-
-    o:delete{match = POST_DELETE}
-    o:grab_extratags{match = POST_EXTRAS}
 
     -- collect main keys
-    local num_mains = 0
+    for k, v in pairs(o.object.tags) do
-    for k, v in pairs(o:tags()) do
+        local ktype = MAIN_KEYS[k]
-        num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
+        if ktype == 'fallback' then
+            if o.has_name then
+                fallback = {k, v, 'named'}
+            end
+        elseif ktype ~= nil then
+            o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
+        end
     end
 
-    if num_mains == 0 then
+    if fallback ~= nil and o.num_entries == 0 then
-        for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do
+        o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS)
-            if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then
+    end
-                return
+end
-            end
-        end
 
-        if fallback ~= nil then
+--------- Convenience functions for simple style configuration -----------------
-            o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS)
+
+
+function module.set_prefilters(data)
+    PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
+    PRE_EXTRAS = module.tag_match{keys = data.extratag_keys,
+                                  tags = data.extratag_tags}
+end
+
+function module.set_main_tags(data)
+    MAIN_KEYS = data
+end
+
+function module.set_name_tags(data)
+    NAMES = module.tag_group(data)
+end
+
+function module.set_address_tags(data)
+    if data.postcode_fallback ~= nil then
+        POSTCODE_FALLBACK = data.postcode_fallback
+        data.postcode_fallback = nil
+    end
+
+    ADDRESS_TAGS = module.tag_group(data)
+end
+
+function module.set_unused_handling(data)
+    if data.extra_keys == nil and data.extra_tags == nil then
+        POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
+        SAVE_EXTRA_MAINS = function() return true end
+    elseif data.delete_keys == nil and data.delete_tags == nil then
+        POST_DELETE = nil
+        SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags}
+    else
+        error("unused handler can have only 'extra_keys' or 'delete_keys' set.")
+    end
+end
+
+function set_relation_types(data)
+    module.RELATION_TYPES = {}
+    for k, v in data do
+        if v == 'multipolygon' then
+            module.RELATION_TYPES[k] = module.relation_as_multipolygon
+        elseif v == 'multiline' then
+            module.RELATION_TYPES[k] = module.relation_as_multiline
         end
     end
 end
 
-
+return module

settings/icu_tokenizer.yaml

@@ -24,6 +24,7 @@ transliteration:
     - ":: lower ()"
     - "[^a-z0-9[:Space:]] >"
     - ":: NFC ()"
+    - "[:Space:]+ > ' '"
 sanitizers:
     - step: clean-housenumbers
       filter-kind:
@@ -37,6 +38,7 @@ sanitizers:
       default-pattern: "[A-Z0-9- ]{3,12}"
     - step: clean-tiger-tags
     - step: split-name-list
+      delimiters: ;
     - step: strip-brace-terms
     - step: tag-analyzer-by-language
       filter-kind: [".*name.*"]

settings/import-address.lua

@@ -0,0 +1,67 @@
+flex = require('flex-base')
+
+flex.set_main_tags{
+    highway = {'always',
+               street_lamp = 'named',
+               traffic_signals = 'named',
+               service = 'named',
+               cycleway = 'named',
+               path = 'named',
+               footway = 'named',
+               steps = 'named',
+               bridleway = 'named',
+               track = 'named',
+               motorway_link = 'named',
+               trunk_link = 'named',
+               primary_link = 'named',
+               secondary_link = 'named',
+               tertiary_link = 'named'},
+    boundary = {administrative = 'named',
+                postal_code = 'always'},
+    landuse = 'fallback',
+    place = 'always'
+}
+
+flex.set_prefilters{delete_keys = {'building', 'source',
+                                   'source', '*source', 'type',
+                                   'is_in:postcode', '*:wikidata', '*:wikipedia',
+                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                                   'name:etymology', 'name:signed', 'name:botanical',
+                                   'addr:street:name', 'addr:street:type'},
+                    delete_tags = {highway = {'no', 'turning_circle', 'mini_roundabout',
+                                              'noexit', 'crossing', 'give_way', 'stop'},
+                                   landuse = {'cemetry', 'no'},
+                                   boundary = {'place'}},
+                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
+                   }
+
+flex.set_name_tags{main = {'name', 'name:*',
+                          'int_name', 'int_name:*',
+                          'nat_name', 'nat_name:*',
+                          'reg_name', 'reg_name:*',
+                          'loc_name', 'loc_name:*',
+                          'old_name', 'old_name:*',
+                          'alt_name', 'alt_name:*', 'alt_name_*',
+                          'official_name', 'official_name:*',
+                          'place_name', 'place_name:*',
+                          'short_name', 'short_name:*'},
+                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
+                            'loc_ref', 'old_ref',
+                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'},
+                   house = {'addr:housename'}
+                  }
+
+flex.set_address_tags{main = {'addr:housenumber',
+                              'addr:conscriptionnumber',
+                              'addr:streetnumber'},
+                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
+                      postcode = {'postal_code', 'postcode', 'addr:postcode',
+                                  'tiger:zip_left', 'tiger:zip_right'},
+                      country = {'country_code', 'ISO3166-1',
+                                 'addr:country_code', 'is_in:country_code',
+                                 'addr:country', 'is_in:country'},
+                      interpolation = {'addr:interpolation'}
+                     }
+
+
+flex.set_unused_handling{extra_keys = {'place'}}

settings/import-admin.lua

@@ -0,0 +1,44 @@
+local flex = require('flex-base')
+
+flex.set_main_tags{
+    boundary = {administrative = 'named'},
+    landuse = 'fallback',
+    place = 'always'
+}
+
+flex.set_prefilters{delete_keys = {'building', 'source', 'highway',
+                                   'addr:housenumber', 'addr:street', 'addr:city',
+                                   'source', '*source', 'type',
+                                   'is_in:postcode', '*:wikidata', '*:wikipedia',
+                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                                   'name:etymology', 'name:signed', 'name:botanical',
+                                   'addr:street:name', 'addr:street:type'},
+                    delete_tags = {landuse = {'cemetry', 'no'},
+                                   boundary = {'place'}},
+                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
+                   }
+
+flex.set_name_tags{main = {'name', 'name:*',
+                          'int_name', 'int_name:*',
+                          'nat_name', 'nat_name:*',
+                          'reg_name', 'reg_name:*',
+                          'loc_name', 'loc_name:*',
+                          'old_name', 'old_name:*',
+                          'alt_name', 'alt_name:*', 'alt_name_*',
+                          'official_name', 'official_name:*',
+                          'place_name', 'place_name:*',
+                          'short_name', 'short_name:*'},
+                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
+                            'loc_ref', 'old_ref',
+                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}
+                  }
+
+flex.set_address_tags{extra = {'addr:*', 'is_in:*'},
+                      postcode = {'postal_code', 'postcode', 'addr:postcode'},
+                      country = {'country_code', 'ISO3166-1',
+                                 'addr:country_code', 'is_in:country_code',
+                                 'addr:country', 'is_in:country'},
+                      postcode_fallback = false
+                     }
+
+flex.set_unused_handling{extra_keys = {'place'}}

settings/import-extratags.lua

@@ -1,13 +1,9 @@
-require('flex-base')
+flex = require('flex-base')
 
-RELATION_TYPES = {
+flex.set_main_tags{
-    multipolygon = relation_as_multipolygon,
+    building = 'fallback',
-    boundary = relation_as_multipolygon,
-    waterway = relation_as_multiline
-}
-
-MAIN_KEYS = {
     emergency = 'always',
+    healthcare = 'fallback',
     historic = 'always',
     military = 'always',
     natural = 'named',
@@ -31,11 +27,13 @@ MAIN_KEYS = {
     man_made = 'always',
     aerialway = 'always',
     boundary = {'named',
-                postal_code = 'named'},
+                postal_code = 'always'},
     aeroway = 'always',
     amenity = 'always',
     club = 'always',
     craft = 'always',
+    junction = 'fallback',
+    landuse = 'fallback',
     leisure = 'always',
     office = 'always',
     mountain_pass = 'always',
@@ -47,55 +45,43 @@ MAIN_KEYS = {
     place = 'always'
 }
 
-MAIN_FALLBACK_KEYS = {
+flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attribution',
-    building = 'named',
+                                   'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
-    landuse = 'named',
+                                   'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
-    junction = 'named',
+                                   'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
-    healthcare = 'named'
+                                   'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
-}
+                                   'ref:linz:*', 'is_in:postcode'},
-
+                    delete_tags = {emergency = {'yes', 'no', 'fire_hydrant'},
-
+                                   historic = {'yes', 'no'},
-PRE_DELETE = tag_match{keys = {'note', 'note:*', 'source', 'source*', 'attribution',
+                                   military = {'yes', 'no'},
-                               'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
+                                   natural = {'yes', 'no', 'coastline'},
-                               'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
+                                   highway = {'no', 'turning_circle', 'mini_roundabout',
-                               'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
+                                              'noexit', 'crossing', 'give_way', 'stop'},
-                               'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
+                                   railway = {'level_crossing', 'no', 'rail'},
-                               'ref:linz:*', 'is_in:postcode'},
+                                   man_made = {'survey_point', 'cutline'},
-                       tags = {emergency = {'yes', 'no', 'fire_hydrant'},
+                                   aerialway = {'pylon', 'no'},
-                               historic = {'yes', 'no'},
+                                   aeroway = {'no'},
-                               military = {'yes', 'no'},
+                                   amenity = {'no'},
-                               natural = {'yes', 'no', 'coastline'},
+                                   club = {'no'},
-                               highway = {'no', 'turning_circle', 'mini_roundabout',
+                                   craft = {'no'},
-                                          'noexit', 'crossing', 'give_way', 'stop'},
+                                   leisure = {'no'},
-                               railway = {'level_crossing', 'no', 'rail'},
+                                   office = {'no'},
-                               man_made = {'survey_point', 'cutline'},
+                                   mountain_pass = {'no'},
-                               aerialway = {'pylon', 'no'},
+                                   shop = {'no'},
-                               aeroway = {'no'},
+                                   tourism = {'yes', 'no'},
-                               amenity = {'no'},
+                                   bridge = {'no'},
-                               club = {'no'},
+                                   tunnel = {'no'},
-                               craft = {'no'},
+                                   waterway = {'riverbank'},
-                               leisure = {'no'},
+                                   building = {'no'},
-                               office = {'no'},
+                                   boundary = {'place'}},
-                               mountain_pass = {'no'},
+                    extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
-                               shop = {'no'},
-                               tourism = {'yes', 'no'},
-                               bridge = {'no'},
-                               tunnel = {'no'},
-                               waterway = {'riverbank'},
-                               building = {'no'},
-                               boundary = {'place'}}
-                      }
-
-POST_DELETE = tag_match{keys = {'tiger:*'}}
-
-PRE_EXTRAS = tag_match{keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
                                'name:etymology', 'name:signed', 'name:botanical',
                                'wikidata', '*:wikidata',
+                               '*:wikipedia', 'brand:wikipedia:*',
                                'addr:street:name', 'addr:street:type'}
-                      }
+                   }
 
-
+flex.set_name_tags{main = {'name', 'name:*',
-NAMES = tag_match{keys = {'name', 'name:*',
                           'int_name', 'int_name:*',
                           'nat_name', 'nat_name:*',
                           'reg_name', 'reg_name:*',
@@ -104,26 +90,24 @@ NAMES = tag_match{keys = {'name', 'name:*',
                           'alt_name', 'alt_name:*', 'alt_name_*',
                           'official_name', 'official_name:*',
                           'place_name', 'place_name:*',
-                          'short_name', 'short_name:*', 'brand'}}
+                          'short_name', 'short_name:*', 'brand'},
+                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
+                            'loc_ref', 'old_ref',
+                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'},
+                   house = {'addr:housename'}
+                  }
 
-REFS = tag_match{keys = {'ref', 'int_ref', 'nat_ref', 'reg_ref', 'loc_ref', 'old_ref',
+flex.set_address_tags{main = {'addr:housenumber',
-                         'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}}
+                              'addr:conscriptionnumber',
-
+                              'addr:streetnumber'},
-POSTCODES = tag_match{keys = {'postal_code', 'postcode', 'addr:postcode',
+                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
-                              'tiger:zip_left', 'tiger:zip_right'}}
+                      postcode = {'postal_code', 'postcode', 'addr:postcode',
-
+                                  'tiger:zip_left', 'tiger:zip_right'},
-COUNTRY_TAGS = tag_match{keys = {'country_code', 'ISO3166-1',
+                      country = {'country_code', 'ISO3166-1',
                                  'addr:country_code', 'is_in:country_code',
-                                 'addr:country', 'is_in:country'}}
+                                 'addr:country', 'is_in:country'},
+                      interpolation = {'addr:interpolation'}
+                     }
 
-HOUSENAME_TAGS = tag_match{keys = {'addr:housename'}}
-
-HOUSENUMBER_TAGS = tag_match{keys = {'addr:housenumber', 'addr:conscriptionnumber',
-                                     'addr:streetnumber'}}
-
-INTERPOLATION_TAGS = tag_match{keys = {'addr:interpolation'}}
-
-ADDRESS_TAGS = tag_match{keys = {'addr:*', 'is_in:*', 'tiger:county'}}
-
-SAVE_EXTRA_MAINS = true
 
+flex.set_unused_handling{delete_keys = {'tiger:*'}}

settings/import-full.lua

@@ -0,0 +1,113 @@
+flex = require('flex-base')
+
+flex.set_main_tags{
+    building = 'fallback',
+    emergency = 'always',
+    healthcare = 'fallback',
+    historic = 'always',
+    military = 'always',
+    natural = 'named',
+    landuse = 'named',
+    highway = {'always',
+               street_lamp = 'named',
+               traffic_signals = 'named',
+               service = 'named',
+               cycleway = 'named',
+               path = 'named',
+               footway = 'named',
+               steps = 'named',
+               bridleway = 'named',
+               track = 'named',
+               motorway_link = 'named',
+               trunk_link = 'named',
+               primary_link = 'named',
+               secondary_link = 'named',
+               tertiary_link = 'named'},
+    railway = 'named',
+    man_made = 'always',
+    aerialway = 'always',
+    boundary = {'named',
+                postal_code = 'always'},
+    aeroway = 'always',
+    amenity = 'always',
+    club = 'always',
+    craft = 'always',
+    junction = 'fallback',
+    landuse = 'fallback',
+    leisure = 'always',
+    office = 'always',
+    mountain_pass = 'always',
+    shop = 'always',
+    tourism = 'always',
+    bridge = 'named_with_key',
+    tunnel = 'named_with_key',
+    waterway = 'named',
+    place = 'always'
+}
+
+flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attribution',
+                                   'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
+                                   'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
+                                   'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
+                                   'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
+                                   'ref:linz:*', 'is_in:postcode',
+                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                                   'name:etymology', 'name:signed', 'name:botanical',
+                                   '*:wikidata', '*:wikipedia', 'brand:wikipedia:*',
+                                   'addr:street:name', 'addr:street:type'},
+                    delete_tags = {emergency = {'yes', 'no', 'fire_hydrant'},
+                                   historic = {'yes', 'no'},
+                                   military = {'yes', 'no'},
+                                   natural = {'yes', 'no', 'coastline'},
+                                   highway = {'no', 'turning_circle', 'mini_roundabout',
+                                              'noexit', 'crossing', 'give_way', 'stop'},
+                                   railway = {'level_crossing', 'no', 'rail'},
+                                   man_made = {'survey_point', 'cutline'},
+                                   aerialway = {'pylon', 'no'},
+                                   aeroway = {'no'},
+                                   amenity = {'no'},
+                                   club = {'no'},
+                                   craft = {'no'},
+                                   leisure = {'no'},
+                                   office = {'no'},
+                                   mountain_pass = {'no'},
+                                   shop = {'no'},
+                                   tourism = {'yes', 'no'},
+                                   bridge = {'no'},
+                                   tunnel = {'no'},
+                                   waterway = {'riverbank'},
+                                   building = {'no'},
+                                   boundary = {'place'}},
+                    extra_keys = {'wikidata', 'wikipedia', 'wikipedia:*'}
+                   }
+
+flex.set_name_tags{main = {'name', 'name:*',
+                          'int_name', 'int_name:*',
+                          'nat_name', 'nat_name:*',
+                          'reg_name', 'reg_name:*',
+                          'loc_name', 'loc_name:*',
+                          'old_name', 'old_name:*',
+                          'alt_name', 'alt_name:*', 'alt_name_*',
+                          'official_name', 'official_name:*',
+                          'place_name', 'place_name:*',
+                          'short_name', 'short_name:*', 'brand'},
+                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
+                            'loc_ref', 'old_ref',
+                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'},
+                   house = {'addr:housename'}
+                  }
+
+flex.set_address_tags{main = {'addr:housenumber',
+                              'addr:conscriptionnumber',
+                              'addr:streetnumber'},
+                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
+                      postcode = {'postal_code', 'postcode', 'addr:postcode',
+                                  'tiger:zip_left', 'tiger:zip_right'},
+                      country = {'country_code', 'ISO3166-1',
+                                 'addr:country_code', 'is_in:country_code',
+                                 'addr:country', 'is_in:country'},
+                      interpolation = {'addr:interpolation'}
+                     }
+
+
+flex.set_unused_handling{extra_keys = {'place'}}

settings/import-street.lua

@@ -0,0 +1,67 @@
+flex = require('flex-base')
+
+flex.set_main_tags{
+    highway = {'always',
+               street_lamp = 'named',
+               traffic_signals = 'named',
+               service = 'named',
+               cycleway = 'named',
+               path = 'named',
+               footway = 'named',
+               steps = 'named',
+               bridleway = 'named',
+               track = 'named',
+               motorway_link = 'named',
+               trunk_link = 'named',
+               primary_link = 'named',
+               secondary_link = 'named',
+               tertiary_link = 'named'},
+    boundary = {administrative = 'named',
+                postal_code = 'always'},
+    landuse = 'fallback',
+    place = 'always'
+}
+
+flex.set_prefilters{delete_keys = {'building', 'source',
+                                   'addr:housenumber', 'addr:street',
+                                   'source', '*source', 'type',
+                                   'is_in:postcode', '*:wikidata', '*:wikipedia',
+                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                                   'name:etymology', 'name:signed', 'name:botanical',
+                                   'addr:street:name', 'addr:street:type'},
+                    delete_tags = {highway = {'no', 'turning_circle', 'mini_roundabout',
+                                              'noexit', 'crossing', 'give_way', 'stop'},
+                                   landuse = {'cemetry', 'no'},
+                                   boundary = {'place'}},
+                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
+                   }
+
+flex.set_name_tags{main = {'name', 'name:*',
+                          'int_name', 'int_name:*',
+                          'nat_name', 'nat_name:*',
+                          'reg_name', 'reg_name:*',
+                          'loc_name', 'loc_name:*',
+                          'old_name', 'old_name:*',
+                          'alt_name', 'alt_name:*', 'alt_name_*',
+                          'official_name', 'official_name:*',
+                          'place_name', 'place_name:*',
+                          'short_name', 'short_name:*'},
+                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
+                            'loc_ref', 'old_ref',
+                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}
+                  }
+
+flex.set_address_tags{main = {'addr:housenumber',
+                              'addr:conscriptionnumber',
+                              'addr:streetnumber'},
+                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
+                      postcode = {'postal_code', 'postcode', 'addr:postcode',
+                                  'tiger:zip_left', 'tiger:zip_right'},
+                      country = {'country_code', 'ISO3166-1',
+                                 'addr:country_code', 'is_in:country_code',
+                                 'addr:country', 'is_in:country'},
+                      interpolation = {'addr:interpolation'},
+                      postcode_fallback = false
+                     }
+
+flex.set_unused_handling{extra_keys = {'place'}}

test/Makefile

@@ -1,12 +1,9 @@
-all: bdd php
+all: bdd php python
 no-test-db: bdd-no-test-db php
 
 bdd:
 	cd bdd && behave -DREMOVE_TEMPLATE=1
 
-icu:
-	cd bdd && behave -DREMOVE_TEMPLATE=1 -DTOKENIZER=icu
-
 php:
 	cd php && phpunit ./
 
@@ -14,4 +11,4 @@ python:
 	pytest python
 
 
-.PHONY: bdd php no-test-db
+.PHONY: bdd php no-test-db python

test/bdd/db/import/interpolation.feature

@@ -29,14 +29,14 @@ Feature: Import of address interpolations
           | N2  | place | house  | 8       |
         And the places
           | osm | class | type   | addr+interpolation | geometry |
-          | W1  | place | houses | even    | 1,2 |
+          | W1  | place | houses | even               | 2,1      |
         And the ways
           | id | nodes |
           | 1  | 2,1 |
         When importing
         Then W1 expands to interpolation
           | start | end | geometry |
-          | 4     | 6   | 8,9 |
+          | 4     | 6   | 9,8      |
 
     Scenario: Simple odd two point interpolation
         Given the grid with origin 1,1
@@ -341,7 +341,7 @@ Feature: Import of address interpolations
         Then W1 expands to interpolation
           | start | end | geometry |
           | 4     | 4   | 144.963016 -37.762946 |
-          | 8     | 8   | 144.963144 -37.7622237 |
+          | 8     | 8   | 144.96314407 -37.762223692 |
 
     Scenario: Place with missing address information
         Given the grid
@@ -456,3 +456,69 @@ Feature: Import of address interpolations
           | foo   |
           | x     |
           | 12-2  |
+
+
+    Scenario: Interpolation line where points have been moved (Github #3022)
+        Given the 0.00001 grid
+         | 1 | | | | | | | | 2 | 3 | 9 | | | | | | | | 4 |
+        Given the places
+          | osm | class | type   | housenr | geometry |
+          | N1  | place | house  | 2       | 1 |
+          | N2  | place | house  | 18      | 3 |
+          | N3  | place | house  | 24      | 9 |
+          | N4  | place | house  | 42      | 4 |
+        And the places
+          | osm | class | type   | addr+interpolation | geometry |
+          | W1  | place | houses | even               | 1,2,3,4  |
+        And the ways
+          | id | nodes   |
+          | 1  | 1,2,3,4 |
+        When importing
+        Then W1 expands to interpolation
+          | start | end |
+          | 4     | 16  |
+          | 20    | 22  |
+          | 26    | 40  |
+
+
+    Scenario: Interpolation line with duplicated points
+        Given the grid
+          | 7 | 10 | 8 | 11 | 9 |
+        Given the places
+          | osm | class | type   | housenr | geometry |
+          | N1  | place | house  | 2       | 7 |
+          | N2  | place | house  | 6       | 8 |
+          | N3  | place | house  | 10      | 8 |
+          | N4  | place | house  | 14      | 9 |
+        And the places
+          | osm | class | type   | addr+interpolation | geometry |
+          | W1  | place | houses | even               | 7,8,8,9  |
+        And the ways
+          | id | nodes   |
+          | 1  | 1,2,3,4 |
+        When importing
+        Then W1 expands to interpolation
+          | start | end | geometry |
+          | 4     | 4   | 10       |
+          | 12    | 12  | 11       |
+
+
+    Scenario: Interpolaton line with broken way geometry (Github #2986)
+        Given the grid
+          | 1 | 8 | 10 | 11 | 9 | 2 | 3 | 4 |
+        Given the places
+          | osm | class | type   | housenr |
+          | N1  | place | house  | 2       |
+          | N2  | place | house  | 8       |
+          | N3  | place | house  | 12      |
+          | N4  | place | house  | 14      |
+        And the places
+          | osm | class | type   | addr+interpolation | geometry |
+          | W1  | place | houses | even               | 8,9      |
+        And the ways
+          | id | nodes       |
+          | 1  | 1,8,9,2,3,4 |
+        When importing
+        Then W1 expands to interpolation
+          | start | end | geometry |
+          | 4     | 6   | 10,11    |

test/bdd/db/import/rank_computation.feature

@@ -255,3 +255,15 @@ Feature: Rank assignment
          | W1     | R10     | True      | 18                  |
          | W1     | R2      | True      | 16                  |
          | W1     | N9      | False     | 18                  |
+
+
+    Scenario: POI nodes with place tags
+        Given the places
+          | osm | class   | type       | name | extratags       |
+          | N23 | amenity | playground | AB   | "place": "city" |
+          | N23 | place   | city       | AB   | "amenity": "playground" |
+        When importing
+        Then placex contains exactly
+          | object      | rank_search | rank_address |
+          | N23:amenity | 30          | 30           |
+          | N23:place   | 16          | 16           |

test/bdd/db/import/search_name.feature

@@ -2,21 +2,26 @@
 Feature: Creation of search terms
     Tests that search_name table is filled correctly
 
-    Scenario Outline: Comma- and semicolon separated names appear as full names
+    Scenario: Semicolon-separated names appear as separate full names
         Given the places
          | osm | class   | type | name+alt_name |
-         | N1  | place   | city | New York<sep>Big Apple |
+         | N1  | place   | city | New York; Big Apple |
         When importing
         Then search_name contains
          | object | name_vector |
          | N1     | #New York, #Big Apple |
 
-    Examples:
+    @fail-legacy
-         | sep |
+    Scenario: Comma-separated names appear as a single full name
-         | ,   |
+        Given the places
-         | ;   |
+         | osm | class   | type | name+alt_name |
+         | N1  | place   | city | New York, Big Apple |
+        When importing
+        Then search_name contains
+         | object | name_vector |
+         | N1     | #New York Big Apple |
 
-    Scenario Outline: Name parts before brackets appear as full names
+    Scenario: Name parts before brackets appear as full names
         Given the places
          | osm | class   | type | name+name |
          | N1  | place   | city | Halle (Saale) |

test/bdd/osm2pgsql/import/tags.feature

@@ -101,6 +101,19 @@ Feature: Tag evaluation
             | N6003  | shop  | -                   |
 
 
+    Scenario: Postcode areas
+        When loading osm data
+            """
+            n1 x12.36853 y51.50618
+            n2 x12.36853 y51.42362
+            n3 x12.63666 y51.42362
+            n4 x12.63666 y51.50618
+            w1 Tboundary=postal_code,ref=3456 Nn1,n2,n3,n4,n1
+            """
+        Then place contains exactly
+            | object | class    | type        | name          |
+            | W1     | boundary | postal_code | 'ref': '3456' |
+
     Scenario: Main with extra
         When loading osm data
             """
@@ -122,10 +135,10 @@ Feature: Tag evaluation
             n8003 Tshop=shoes,name:source=survey
             """
         Then place contains exactly
-            | object | class | extratags    |
+            | object | class | name | extratags    |
-            | N8001  | shop  | 'xx': 'yy'   |
+            | N8001  | shop  |  -   | 'xx': 'yy'   |
-            | N8002  | shop  | 'ele': '234' |
+            | N8002  | shop  |  -   | 'ele': '234' |
-            | N8003  | shop  | -            |
+            | N8003  | shop  |  -   | -            |
 
 
     Scenario: Admin levels

test/php/Nominatim/DebugTest.php

@@ -37,14 +37,14 @@ class DebugTest extends \PHPUnit\Framework\TestCase
 <pre><b>Var1:</b>  <i>True</i></pre>
 <pre><b>Var2:</b>  <i>False</i></pre>
 <pre><b>Var3:</b>  0</pre>
-<pre><b>Var4:</b>  'String'</pre>
+<pre><b>Var4:</b>  &#039;String&#039;</pre>
-<pre><b>Var5:</b>  0 => 'one'
+<pre><b>Var5:</b>  0 => &#039;one&#039;
-       1 => 'two'
+       1 => &#039;two&#039;
-       2 => 'three'</pre>
+       2 => &#039;three&#039;</pre>
-<pre><b>Var6:</b>  'key' => 'value'
+<pre><b>Var6:</b>  &#039;key&#039; => &#039;value&#039;
-       'key2' => 'value2'</pre>
+       &#039;key2&#039; => &#039;value2&#039;</pre>
 <pre><b>Var7:</b>  me as string</pre>
-<pre><b>Var8:</b>  'value', 'value2'</pre>
+<pre><b>Var8:</b>  &#039;value&#039;, &#039;value2&#039;</pre>
 
 EOT
         );
@@ -64,10 +64,10 @@ EOT
     public function testDebugArray()
     {
         $this->expectOutputString(<<<EOT
-<pre><b>Arr0:</b>  'null'</pre>
+<pre><b>Arr0:</b>  &#039;null&#039;</pre>
-<pre><b>Arr1:</b>  'key1' => 'val1'
+<pre><b>Arr1:</b>  &#039;key1&#039; => &#039;val1&#039;
-       'key2' => 'val2'
+       &#039;key2&#039; => &#039;val2&#039;
-       'key3' => 'val3'</pre>
+       &#039;key3&#039; => &#039;val3&#039;</pre>
 
 EOT
         );
@@ -93,12 +93,12 @@ EOT
     <th><small>1</small></th>
   </tr>
   <tr>
-    <td><pre>'one'</pre></td>
+    <td><pre>&#039;one&#039;</pre></td>
-    <td><pre>'two'</pre></td>
+    <td><pre>&#039;two&#039;</pre></td>
   </tr>
   <tr>
-    <td><pre>'three'</pre></td>
+    <td><pre>&#039;three&#039;</pre></td>
-    <td><pre>'four'</pre></td>
+    <td><pre>&#039;four&#039;</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -109,9 +109,9 @@ EOT
     <th><small>key3</small></th>
   </tr>
   <tr>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
 </table>
 
@@ -147,18 +147,18 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'one'</pre></td>
+    <td><pre>&#039;one&#039;</pre></td>
-    <td><pre>'two'</pre></td>
+    <td><pre>&#039;two&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group2</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -171,15 +171,15 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
 </table>
 

test/python/tools/test_add_osm_data.py

@@ -24,10 +24,14 @@ class CaptureGetUrl:
         return '<xml></xml>'
 
 
-def test_import_osm_file_simple(table_factory, osm2pgsql_options, capfd):
+@pytest.fixture(autouse=True)
-    table_factory('place', content=((1, ), ))
+def setup_delete_postprocessing(temp_db_cursor):
+    temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION flush_deleted_places()
+                              RETURNS INTEGER AS $$ SELECT 1 $$ LANGUAGE SQL""")
 
-    assert add_osm_data.add_data_from_file(Path('change.osm'), osm2pgsql_options) == 0
+def test_import_osm_file_simple(dsn, table_factory, osm2pgsql_options, capfd):
+
+    assert add_osm_data.add_data_from_file(dsn, Path('change.osm'), osm2pgsql_options) == 0
     captured = capfd.readouterr()
 
     assert '--append' in captured.out
@@ -41,11 +45,11 @@ def test_import_osm_file_simple(table_factory, osm2pgsql_options, capfd):
 @pytest.mark.parametrize("osm_type", ['node', 'way', 'relation'])
 @pytest.mark.parametrize("main_api,url", [(True, 'https://www.openstreetmap.org/api'),
                                           (False, 'https://overpass-api.de/api/interpreter?')])
-def test_import_osm_object_main_api(osm2pgsql_options, monkeypatch, capfd,
+def test_import_osm_object_main_api(dsn, osm2pgsql_options, monkeypatch,
-                                    osm_type, main_api, url):
+                                    capfd, osm_type, main_api, url):
     get_url_mock = CaptureGetUrl(monkeypatch)
 
-    add_osm_data.add_osm_object(osm_type, 4536, main_api, osm2pgsql_options)
+    add_osm_data.add_osm_object(dsn, osm_type, 4536, main_api, osm2pgsql_options)
     captured = capfd.readouterr()
 
     assert get_url_mock.url.startswith(url)

test/python/tools/test_refresh.py

@@ -48,7 +48,7 @@ def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, r
 def test_recompute_importance(placex_table, table_factory, temp_db_conn, temp_db_cursor):
     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE,
                                               country_code varchar(2),
-                                              osm_type varchar(1), osm_id BIGINT,
+                                              rank_search SMALLINT,
                                               centroid GEOMETRY,
                                               OUT importance FLOAT,
                                               OUT wikipedia TEXT)

vagrant/Install-on-Ubuntu-18.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-10-postgis-2.4 \
                         postgresql-contrib-10 postgresql-10-postgis-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-pip \
-                        python3-psutil python3-jinja2 python3-yaml python3-icu
+                        python3-psutil python3-jinja2 python3-yaml python3-icu git
 
 # Some of the Python packages that come with Ubuntu 18.04 are too old, so
 # install the latest version from pip:
@@ -105,18 +105,25 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.2.4.tar.bz2
+    git clone --recursive https://github.com/openstreetmap/Nominatim.git
-    tar xf Nominatim-4.2.4.tar.bz2
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.2.4
+    cmake $USERHOME/Nominatim
     make
     sudo make install
 

vagrant/Install-on-Ubuntu-20.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-contrib-12 postgresql-12-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \
                         python3-psycopg2 python3-psutil python3-jinja2 \
-                        python3-icu python3-datrie python3-yaml
+                        python3-icu python3-datrie python3-yaml git
 
 #
 # System Configuration
@@ -99,18 +99,25 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.2.4.tar.bz2
+    git clone --recursive https://github.com/openstreetmap/Nominatim.git
-    tar xf Nominatim-4.2.4.tar.bz2
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.2.4
+    cmake $USERHOME/Nominatim
     make
     sudo make install
 

vagrant/Install-on-Ubuntu-22.sh

@@ -28,7 +28,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
                         postgresql-contrib-14 postgresql-14-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \
                         python3-psycopg2 python3-psutil python3-jinja2 \
-                        python3-icu python3-datrie
+                        python3-icu python3-datrie git
 
 #
 # System Configuration
@@ -99,18 +99,25 @@ fi                                    #DOCS:
 #
 if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
     cd $USERHOME
-    wget https://nominatim.org/release/Nominatim-4.2.4.tar.bz2
+    git clone --recursive https://github.com/openstreetmap/Nominatim.git
-    tar xf Nominatim-4.2.4.tar.bz2
+    cd Nominatim
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
 
+# When installing the latest source from github, you also need to
+# download the country grid:
+
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
+    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
+fi                                 #DOCS:
+
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
     mkdir $USERHOME/build
     cd $USERHOME/build
-    cmake $USERHOME/Nominatim-4.2.4
+    cmake $USERHOME/Nominatim
     make
     sudo make install