prepare release 4.2.2

The full style should only save the necessary tags needed for
processing.

.github/workflows/ci-tests.yml

@@ -69,8 +69,10 @@ jobs:
               uses: shivammathur/setup-php@v2
               with:
                   php-version: ${{ matrix.php }}
-                  tools: phpunit, phpcs, composer
+                  tools: phpunit:9, phpcs, composer
                   ini-values: opcache.jit=disable
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
             - uses: actions/setup-python@v4
               with:
@@ -100,18 +102,22 @@ jobs:
 
             - name: Install latest pylint/mypy
               run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions
+              if: matrix.ubuntu == 22
 
             - name: PHP linting
               run: phpcs --report-width=120 .
               working-directory: Nominatim
+              if: matrix.ubuntu == 22
 
             - name: Python linting
               run: pylint nominatim
               working-directory: Nominatim
+              if: matrix.ubuntu == 22
 
             - name: Python static typechecking
               run: mypy --strict nominatim
               working-directory: Nominatim
+              if: matrix.ubuntu == 22
 
 
             - name: PHP unit tests

.pylintrc

@@ -13,6 +13,6 @@ ignored-classes=NominatimArgs,closing
 # 'too-many-ancestors' is triggered already by deriving from UserDict
 # 'not-context-manager' disabled because it causes false positives once
 #   typed Python is enabled. See also https://github.com/PyCQA/pylint/issues/5273
-disable=too-few-public-methods,duplicate-code,too-many-ancestors,bad-option-value,no-self-use,not-context-manager
+disable=too-few-public-methods,duplicate-code,too-many-ancestors,bad-option-value,no-self-use,not-context-manager,use-dict-literal
 
 good-names=i,x,y,m,fd,db,cc

CMakeLists.txt

@@ -20,7 +20,7 @@ project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 4)
 set(NOMINATIM_VERSION_MINOR 2)
-set(NOMINATIM_VERSION_PATCH 0)
+set(NOMINATIM_VERSION_PATCH 2)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 
@@ -269,6 +269,12 @@ install(FILES settings/env.defaults
               settings/import-address.style
               settings/import-full.style
               settings/import-extratags.style
+              settings/import-admin.lua
+              settings/import-street.lua
+              settings/import-address.lua
+              settings/import-full.lua
+              settings/import-extratags.lua
+              settings/flex-base.lua
               settings/icu_tokenizer.yaml
               settings/country_settings.yaml
         DESTINATION ${NOMINATIM_CONFIGDIR})

ChangeLog

@@ -1,3 +1,22 @@
+4.2.2
+
+ * extend flex-style library to fully support all default styles
+ * fix handling of Hebrew aleph
+ * do not assign postcodes to rivers
+ * fix string matching in PHP code
+ * update osm2pgsql (various updates to flex)
+ * fix slow query when deleting places on update
+ * fix CLI details query
+ * fix recalculation of importance values
+ * fix polygon simplification in reverse results
+ * add class/type information to reverse geocodejson result
+ * minor improvements to default tokenizer configuration
+ * various smaller fixes to documentation
+
+4.2.1
+
+ * fix XSS vulnerability in debug view
+
 4.2.0
 
  * add experimental support for osm2pgsql flex style
@@ -21,6 +40,10 @@
  * typing fixes to work with latest type annotations from typeshed
  * smaller improvements to documentation (thanks to @mausch)
 
+4.1.1
+
+ * fix XSS vulnerability in debug view
+
 4.1.0
 
  * switch to ICU tokenizer as default
@@ -57,6 +80,10 @@
  * add setup instructions for updates and systemd
  * drop support for PostgreSQL 9.5
 
+4.0.2
+
+ * fix XSS vulnerability in debug view
+
 4.0.1
 
  * fix initialisation error in replication script
@@ -95,6 +122,10 @@
  * add testing of installation scripts via CI
  * drop support for Python < 3.6 and Postgresql < 9.5
 
+3.7.3
+
+ * fix XSS vulnerability in debug view
+
 3.7.2
 
  * fix database check for reverse-only imports

VAGRANT.md

@@ -1,6 +1,6 @@
 # Install Nominatim in a virtual machine for development and testing
 
-This document describes how you can install Nominatim inside a Ubuntu 16
+This document describes how you can install Nominatim inside a Ubuntu 22
 virtual machine on your desktop/laptop (host machine). The goal is to give
 you a development environment to easily edit code and run the test suite
 without affecting the rest of your system. 
@@ -69,8 +69,7 @@ installation.
 PHP errors are written to `/var/log/apache2/error.log`.
 
 With `echo` and `var_dump()` you write into the output (HTML/XML/JSON) when
-you either add `&debug=1` to the URL (preferred) or set
+you either add `&debug=1` to the URL.
-`@define('CONST_Debug', true);` in `settings/local.php`.
 
 In the Python BDD test you can use `logger.info()` for temporary debug
 statements.
@@ -130,6 +129,10 @@ and then
 Yes, Vagrant and Virtualbox can be installed on MS Windows just fine. You need a 64bit
 version of Windows.
 
+##### Will it run on Apple Silicon?
+
+You might need to replace Virtualbox with [Parallels](https://www.parallels.com/products/desktop/).
+There is no free/open source version of Parallels.
 
 ##### Why Monaco, can I use another country?
 
@@ -141,11 +144,12 @@ No. Long running Nominatim installations will differ once new import features (o
 bug fixes) get added since those usually only get applied to new/changed data.
 
 Also this document skips the optional Wikipedia data import which affects ranking
-of search results. See [Nominatim installation](https://nominatim.org/release-docs/latest/admin/Installation) for details.
+of search results. See [Nominatim installation](https://nominatim.org/release-docs/latest/admin/Installation)
+for details.
 
 ##### Why Ubuntu? Can I test CentOS/Fedora/CoreOS/FreeBSD?
 
-There is a Vagrant script for CentOS available, but the Nominatim directory
+There used to be a Vagrant script for CentOS available, but the Nominatim directory
 isn't symlinked/mounted to the host which makes development trickier. We used
 it mainly for debugging installation with SELinux.
 
@@ -154,14 +158,17 @@ are slightly different, e.g. the name of the package manager, Apache2 package
 name, location of files. We chose Ubuntu because that is closest to the
 nominatim.openstreetmap.org production environment.
 
-You can configure/download other Vagrant boxes from [https://app.vagrantup.com/boxes/search](https://app.vagrantup.com/boxes/search).
+You can configure/download other Vagrant boxes from
+[https://app.vagrantup.com/boxes/search](https://app.vagrantup.com/boxes/search).
 
 ##### How can I connect to an existing database?
 
-Let's say you have a Postgres database named `nominatim_it` on server `your-server.com` and port `5432`. The Postgres username is `postgres`. You can edit `settings/local.php` and point Nominatim to it.
+Let's say you have a Postgres database named `nominatim_it` on server `your-server.com`
+and port `5432`. The Postgres username is `postgres`. You can edit the `.env` in your
+project directory and point Nominatim to it.
+
+    NOMINATIM_DATABASE_DSN="pgsql:host=your-server.com;port=5432;user=postgres;dbname=nominatim_it
 
-    pgsql:host=your-server.com;port=5432;user=postgres;dbname=nominatim_it
-    
 No data import or restarting necessary.
 
 If the Postgres installation is behind a firewall, you can try
@@ -169,11 +176,12 @@ If the Postgres installation is behind a firewall, you can try
     ssh -L 9999:localhost:5432 your-username@your-server.com
 
 inside the virtual machine. It will map the port to `localhost:9999` and then
-you edit `settings/local.php` with
+you edit `.env` file with
 
-    @define('CONST_Database_DSN', 'pgsql:host=localhost;port=9999;user=postgres;dbname=nominatim_it');
+    NOMINATIM_DATABASE_DSN="pgsql:host=localhost;port=9999;user=postgres;dbname=nominatim_it"
 
-To access postgres directly remember to specify the hostname, e.g. `psql --host localhost --port 9999 nominatim_it`
+To access postgres directly remember to specify the hostname,
+e.g. `psql --host localhost --port 9999 nominatim_it`
 
 
 ##### My computer is slow and the import takes too long. Can I start the virtual machine "in the cloud"?

docs/admin/Import.md

@@ -74,7 +74,7 @@ but it will improve the quality of the results if this is installed.
 This data is available as a binary download. Put it into your project directory:
 
     cd $PROJECT_DIR
-    wget https://www.nominatim.org/data/wikimedia-importance.sql.gz
+    wget https://nominatim.org/data/wikimedia-importance.sql.gz
 
 The file is about 400MB and adds around 4GB to the Nominatim database.
 
@@ -92,8 +92,8 @@ and the UK (using the [CodePoint OpenData set](https://osdatahub.os.uk/downloads
 This data can be optionally downloaded into the project directory:
 
     cd $PROJECT_DIR
-    wget https://www.nominatim.org/data/gb_postcodes.csv.gz
+    wget https://nominatim.org/data/gb_postcodes.csv.gz
-    wget https://www.nominatim.org/data/us_postcodes.csv.gz
+    wget https://nominatim.org/data/us_postcodes.csv.gz
 
 You can also add your own custom postcode sources, see
 [Customization of postcodes](../customize/Postcodes.md).

docs/admin/Installation.md

@@ -135,7 +135,7 @@ git clone --recursive https://github.com/openstreetmap/Nominatim.git
 The development version does not include the country grid. Download it separately:
 
 ```
-wget -O Nominatim/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
 ```
 
 ### Building Nominatim

docs/api/Faq.md

@@ -59,3 +59,27 @@ suited for these kinds of queries.
 
 That said if you installed your own Nominatim instance you can use the
 `nominatim export` PHP script as basis to return such lists.
+
+#### 7. My result has a wrong postcode. Where does it come from?
+
+Most places in OSM don't have a postcode, so Nominatim tries to interpolate
+one. It first look at all the places that make up the address of the place.
+If one of them has a postcode defined, this is the one to be used. When
+none of the address parts has a postcode either, Nominatim interpolates one
+from the surrounding objects. If the postcode is for your result is one, then
+most of the time there is an OSM object with the wrong postcode nearby.
+
+To find the bad postcode, go to
+[https://nominatim.openstreetmap.org](https://nominatim.openstreetmap.org)
+and search for your place. When you have found it, click on the 'details' link
+under the result to go to the details page. There is a field 'Computed Postcode'
+which should display the bad postcode. Click on the 'how?' link. A small
+explanation text appears. It contains a link to a query for Overpass Turbo.
+Click on that and you get a map with all places in the area that have the bad
+postcode. If none is displayed, zoom the map out a bit and then click on 'Run'.
+
+Now go to [OpenStreetMap](https://openstreetmap.org) and fix the error you
+have just found. It will take at least a day for Nominatim to catch up with
+your data fix. Sometimes longer, depending on how much editing activity is in
+the area.
+

docs/api/Output.md

@@ -211,8 +211,8 @@ be more than one. The attributes of that element contain:
  * `ref` - content of `ref` tag if it exists
  * `lat`, `lon` - latitude and longitude of the centroid of the object
  * `boundingbox` - comma-separated list of corner coordinates ([see notes](#boundingbox))
- * `place_rank` - class [search rank](../customize/Ranking#search-rank)
+ * `place_rank` - class [search rank](../customize/Ranking.md#search-rank)
- * `address_rank` - place [address rank](../customize/Ranking#address-rank)
+ * `address_rank` - place [address rank](../customize/Ranking.md#address-rank)
  * `display_name` - full comma-separated address
  * `class`, `type` - key and value of the main OSM tag
  * `importance` - computed importance rank

docs/api/Reverse.md

@@ -35,7 +35,7 @@ Additional parameters are accepted as listed below.
 
 !!! warning "Deprecation warning"
     The reverse API used to allow address lookup for a single OSM object by
-    its OSM id. This use is now deprecated. Use the [Address Lookup API](../Lookup)
+    its OSM id. This use is now deprecated. Use the [Address Lookup API](Lookup.md)
     instead.
 
 ### Output format

docs/customize/Importance.md

@@ -12,7 +12,7 @@ customize them.
 The main value for importance is derived from page ranking values for Wikipedia
 pages for a place. For places that do not have their own
 Wikipedia page, a formula is used that derives a static importance from the
-places [search rank](../customize/Ranking#search-rank).
+places [search rank](../customize/Ranking.md#search-rank).
 
 In a second step, a secondary importance value is added which is meant to
 represent how well-known the general area is where the place is located. It

docs/customize/Settings.md

@@ -666,7 +666,7 @@ The entries in the log file have the following format:
     <request time> <execution time in s> <number of results> <type> "<query string>"
 
 Request time is the time when the request was started. The execution time is
-given in ms and corresponds to the time the query took executing in PHP.
+given in seconds and corresponds to the time the query took executing in PHP.
 type contains the name of the endpoint used.
 
 Can be used as the same time as NOMINATIM_LOG_DB.

lib-php/DebugHtml.php

@@ -135,7 +135,7 @@ class Debug
 
     public static function printSQL($sSQL)
     {
-        echo '<p><tt><font color="#aaa">'.$sSQL.'</font></tt></p>'."\n";
+        echo '<p><tt><font color="#aaa">'.htmlspecialchars($sSQL, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401).'</font></tt></p>'."\n";
     }
 
     private static function outputVar($mVar, $sPreNL)
@@ -178,11 +178,12 @@ class Debug
         }
 
         if (is_string($mVar)) {
-            echo "'$mVar'";
+            $sOut = "'$mVar'";
-            return strlen($mVar) + 2;
+        } else {
+            $sOut = (string)$mVar;
         }
 
-        echo (string)$mVar;
+        echo htmlspecialchars($sOut, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401);
-        return strlen((string)$mVar);
+        return strlen($sOut);
     }
 }

lib-php/Geocode.php

@@ -874,7 +874,7 @@ class Geocode
                 $iCountWords = 0;
                 $sAddress = $aResult['langaddress'];
                 foreach ($aRecheckWords as $i => $sWord) {
-                    if (stripos($sAddress, $sWord)!==false) {
+                    if (grapheme_stripos($sAddress, $sWord)!==false) {
                         $iCountWords++;
                         if (preg_match('/(^|,)\s*'.preg_quote($sWord, '/').'\s*(,|$)/', $sAddress)) {
                             $iCountWords += 0.1;

lib-php/PlaceLookup.php

@@ -524,12 +524,7 @@ class PlaceLookup
 
         // Get the bounding box and outline polygon
         $sSQL = 'select place_id,0 as numfeatures,st_area(geometry) as area,';
-        if ($fLonReverse != null && $fLatReverse != null) {
+        $sSQL .= ' ST_Y(centroid) as centrelat, ST_X(centroid) as centrelon,';
-            $sSQL .= ' ST_Y(closest_point) as centrelat,';
-            $sSQL .= ' ST_X(closest_point) as centrelon,';
-        } else {
-            $sSQL .= ' ST_Y(centroid) as centrelat, ST_X(centroid) as centrelon,';
-        }
         $sSQL .= ' ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,';
         $sSQL .= ' ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon';
         if ($this->bIncludePolygonAsGeoJSON) {
@@ -544,19 +539,21 @@ class PlaceLookup
         if ($this->bIncludePolygonAsText) {
             $sSQL .= ',ST_AsText(geometry) as astext';
         }
+
+        $sSQL .= ' FROM (SELECT place_id';
         if ($fLonReverse != null && $fLatReverse != null) {
-            $sFrom = ' from (SELECT * , CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
+            $sSQL .= ',CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
-            $sFrom .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
+            $sSQL .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
-            $sFrom .=' ELSE centroid END AS closest_point';
+            $sSQL .=' ELSE centroid END AS centroid';
-            $sFrom .= ' from placex where place_id = '.$iPlaceID.') as plx';
         } else {
-            $sFrom = ' from placex where place_id = '.$iPlaceID;
+            $sSQL .= ',centroid';
         }
         if ($this->fPolygonSimplificationThreshold > 0) {
-            $sSQL .= ' from (select place_id,centroid,ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry'.$sFrom.') as plx';
+            $sSQL .= ',ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry';
         } else {
-            $sSQL .= $sFrom;
+            $sSQL .= ',geometry';
         }
+        $sSQL .= ' FROM placex where place_id = '.$iPlaceID.') as plx';
 
         $aPointPolygon = $this->oDB->getRow($sSQL, null, 'Could not get outline');
 

lib-php/template/address-geocodejson.php

@@ -36,6 +36,9 @@ if (empty($aPlace)) {
         $aFilteredPlaces['properties']['geocoding']['osm_id'] = $aPlace['osm_id'];
     }
 
+    $aFilteredPlaces['properties']['geocoding']['osm_key'] = $aPlace['class'];
+    $aFilteredPlaces['properties']['geocoding']['osm_value'] = $aPlace['type'];
+
     $aFilteredPlaces['properties']['geocoding']['type'] = addressRankToGeocodeJsonType($aPlace['rank_address']);
 
     $aFilteredPlaces['properties']['geocoding']['accuracy'] = (int) $fDistance;

lib-sql/functions/place_triggers.sql

@@ -384,7 +384,19 @@ BEGIN
 
   -- Mark for delete in the placex table
   UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
-    WHERE placex.osm_type = place_to_be_deleted.osm_type
+    WHERE placex.osm_type = 'N' and place_to_be_deleted.osm_type = 'N'
+          and placex.osm_id = place_to_be_deleted.osm_id
+          and placex.class = place_to_be_deleted.class
+          and placex.type = place_to_be_deleted.type
+          and not deferred;
+  UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
+    WHERE placex.osm_type = 'W' and place_to_be_deleted.osm_type = 'W'
+          and placex.osm_id = place_to_be_deleted.osm_id
+          and placex.class = place_to_be_deleted.class
+          and placex.type = place_to_be_deleted.type
+          and not deferred;
+  UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
+    WHERE placex.osm_type = 'R' and place_to_be_deleted.osm_type = 'R'
           and placex.osm_id = place_to_be_deleted.osm_id
           and placex.class = place_to_be_deleted.class
           and placex.type = place_to_be_deleted.type

lib-sql/functions/placex_triggers.sql

@@ -1120,7 +1120,7 @@ BEGIN
   ELSE
     -- No linked place? As a last resort check if the boundary is tagged with
     -- a place type and adapt the rank address.
-    IF NEW.rank_address > 0 and NEW.extratags ? 'place' THEN
+    IF NEW.rank_address between 4 and 25 and NEW.extratags ? 'place' THEN
       SELECT address_rank INTO place_address_level
         FROM compute_place_rank(NEW.country_code, 'A', 'place',
                                 NEW.extratags->'place', 0::SMALLINT, False, null);
@@ -1230,7 +1230,11 @@ BEGIN
     {% endif %}
   END IF;
 
-  IF NEW.postcode is null AND NEW.rank_search > 8 THEN
+  IF NEW.postcode is null AND NEW.rank_search > 8
+     AND (NEW.rank_address > 0
+          OR ST_GeometryType(NEW.geometry) not in ('ST_LineString','ST_MultiLineString')
+          OR ST_Length(NEW.geometry) < 0.02)
+  THEN
     NEW.postcode := get_nearest_postcode(NEW.country_code, NEW.geometry);
   END IF;
 

nominatim/clicmd/api.py

@@ -248,9 +248,9 @@ class APIDetails:
         if args.node:
             params = dict(osmtype='N', osmid=args.node)
         elif args.way:
-            params = dict(osmtype='W', osmid=args.node)
+            params = dict(osmtype='W', osmid=args.way)
         elif args.relation:
-            params = dict(osmtype='R', osmid=args.node)
+            params = dict(osmtype='R', osmid=args.relation)
         else:
             params = dict(place_id=args.place_id)
         if args.object_class:

nominatim/tools/exec_utils.py

@@ -118,7 +118,7 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
     cmd = [str(options['osm2pgsql']),
            '--hstore', '--latlon', '--slim',
            '--log-progress', 'true',
-           '--number-processes', str(options['threads']),
+           '--number-processes', '1' if options['append'] else str(options['threads']),
            '--cache', str(options['osm2pgsql_cache']),
            '--style', str(options['osm2pgsql_style'])
           ]

nominatim/tools/refresh.py

@@ -176,7 +176,7 @@ def recompute_importance(conn: Connection) -> None:
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance
-                FROM compute_importance(extratags, country_code, osm_type, osm_id, centroid))
+                FROM compute_importance(extratags, country_code, rank_search, centroid))
             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance

nominatim/version.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2022 by the Nominatim developer community.
+# Copyright (C) 2023 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Version information for Nominatim.
@@ -25,7 +25,7 @@ from typing import Optional, Tuple
 # patch level when cherry-picking the commit with the migration.
 #
 # Released versions always have a database patch level of 0.
-NOMINATIM_VERSION = (4, 2, 0, 0)
+NOMINATIM_VERSION = (4, 2, 2, 0)
 
 POSTGRESQL_REQUIRED_VERSION = (9, 6)
 POSTGIS_REQUIRED_VERSION = (2, 2)

settings/flex-base.lua

@@ -1,9 +1,19 @@
 -- Core functions for Nominatim import flex style.
 --
 
+local module = {}
+
+local PRE_DELETE = nil
+local PRE_EXTRAS = nil
+local MAIN_KEYS = nil
+local NAMES = nil
+local ADDRESS_TAGS = nil
+local SAVE_EXTRA_MAINS = false
+local POSTCODE_FALLBACK = true
+
 
 -- The single place table.
-place_table = osm2pgsql.define_table{
+local place_table = osm2pgsql.define_table{
     name = "place",
     ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
     columns = {
@@ -14,7 +24,25 @@ place_table = osm2pgsql.define_table{
         { column = 'address', type = 'hstore' },
         { column = 'extratags', type = 'hstore' },
         { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
-    }
+    },
+    indexes = {}
+}
+
+------------ Geometry functions for relations ---------------------
+
+function module.relation_as_multipolygon(o)
+    return o:as_multipolygon()
+end
+
+function module.relation_as_multiline(o)
+    return o:as_multilinestring():line_merge()
+end
+
+
+module.RELATION_TYPES = {
+    multipolygon = module.relation_as_multipolygon,
+    boundary = module.relation_as_multipolygon,
+    waterway = module.relation_as_multiline
 }
 
 ------------- Place class ------------------------------------------
@@ -43,6 +71,17 @@ function Place.new(object, geom_func)
     return self
 end
 
+function Place:clean(data)
+    for k, v in pairs(self.object.tags) do
+        if data.delete ~= nil and data.delete(k, v) then
+            self.object.tags[k] = nil
+        elseif data.extra ~= nil and data.extra(k, v) then
+            self.extratags[k] = v
+            self.object.tags[k] = nil
+        end
+    end
+end
+
 function Place:delete(data)
     if data.match ~= nil then
         for k, v in pairs(self.object.tags) do
@@ -69,54 +108,37 @@ function Place:grab_extratags(data)
     return count
 end
 
-function Place:grab_address(data)
+local function strip_address_prefix(k)
+    if k:sub(1, 5) == 'addr:' then
+        return k:sub(6)
+    end
+
+    if k:sub(1, 6) == 'is_in:' then
+        return k:sub(7)
+    end
+
+    return k
+end
+
+
+function Place:grab_address_parts(data)
     local count = 0
 
-    if data.match ~= nil then
+    if data.groups ~= nil then
         for k, v in pairs(self.object.tags) do
-            if data.match(k, v) then
+            local atype = data.groups(k, v)
-                self.object.tags[k] = nil
 
-                if data.include_on_name == true then
+            if atype ~= nil then
+                if atype == 'main' then
                     self.has_name = true
-                end
+                    self.address[strip_address_prefix(k)] = v
-
+                    count = count + 1
-                if data.out_key ~= nil then
+                elseif atype == 'extra' then
-                    self.address[data.out_key] = v
+                    self.address[strip_address_prefix(k)] = v
-                    return 1
-                end
-
-                if k:sub(1, 5) == 'addr:' then
-                    self.address[k:sub(6)] = v
-                elseif k:sub(1, 6) == 'is_in:' then
-                    self.address[k:sub(7)] = v
                 else
-                    self.address[k] = v
+                    self.address[atype] = v
                 end
-                count = count + 1
-            end
-        end
-    end
-
-    return count
-end
-
-function Place:set_address(key, value)
-    self.address[key] = value
-end
-
-function Place:grab_name(data)
-    local count = 0
-
-    if data.match ~= nil then
-        for k, v in pairs(self.object.tags) do
-            if data.match(k, v) then
                 self.object.tags[k] = nil
-                self.names[k] = v
-                if data.include_on_name ~= false then
-                    self.has_name = true
-                end
-                count = count + 1
             end
         end
     end
@@ -124,13 +146,30 @@ function Place:grab_name(data)
     return count
 end
 
-function Place:grab_tag(key)
+
-    return self.object:grab_tag(key)
+function Place:grab_name_parts(data)
+    local fallback = nil
+
+    if data.groups ~= nil then
+        for k, v in pairs(self.object.tags) do
+            local atype = data.groups(k, v)
+
+            if atype ~= nil then
+                self.names[k] = v
+                self.object.tags[k] = nil
+                if atype == 'main' then
+                    self.has_name = true
+                elseif atype == 'house' then
+                    self.has_name = true
+                    fallback = {'place', 'house', 'always'}
+                end
+            end
+        end
+    end
+
+    return fallback
 end
 
-function Place:tags()
-    return self.object.tags
-end
 
 function Place:write_place(k, v, mtype, save_extra_mains)
     if mtype == nil then
@@ -184,9 +223,9 @@ function Place:write_row(k, v, save_extra_mains)
         return 0
     end
 
-    if save_extra_mains then
+    if save_extra_mains ~= nil then
         for extra_k, extra_v in pairs(self.object.tags) do
-            if extra_k ~= k then
+            if extra_k ~= k and save_extra_mains(extra_k, extra_v) then
                 self.extratags[extra_k] = extra_v
             end
         end
@@ -204,7 +243,9 @@ function Place:write_row(k, v, save_extra_mains)
 
     if save_extra_mains then
         for k, v in pairs(self.object.tags) do
-            self.extratags[k] = nil
+            if save_extra_mains(k, v) then
+                self.extratags[k] = nil
+            end
         end
     end
 
@@ -214,7 +255,7 @@ function Place:write_row(k, v, save_extra_mains)
 end
 
 
-function tag_match(data)
+function module.tag_match(data)
     if data == nil or next(data) == nil then
         return nil
     end
@@ -276,17 +317,72 @@ function tag_match(data)
 end
 
 
+function module.tag_group(data)
+    if data == nil or next(data) == nil then
+        return nil
+    end
+
+    local fullmatches = {}
+    local key_prefixes = {}
+    local key_suffixes = {}
+
+    for group, tags in pairs(data) do
+        for _, key in pairs(tags) do
+            if key:sub(1, 1) == '*' then
+                if #key > 1 then
+                    if key_suffixes[#key - 1] == nil then
+                        key_suffixes[#key - 1] = {}
+                    end
+                    key_suffixes[#key - 1][key:sub(2)] = group
+                end
+            elseif key:sub(#key, #key) == '*' then
+                if key_prefixes[#key - 1] == nil then
+                    key_prefixes[#key - 1] = {}
+                end
+                key_prefixes[#key - 1][key:sub(1, #key - 1)] = group
+            else
+                fullmatches[key] = group
+            end
+        end
+    end
+
+    return function (k, v)
+        local val = fullmatches[k]
+        if val ~= nil then
+            return val
+        end
+
+        for slen, slist in pairs(key_suffixes) do
+            if #k >= slen then
+                val = slist[k:sub(-slen)]
+                if val ~= nil then
+                    return val
+                end
+            end
+        end
+
+        for slen, slist in pairs(key_prefixes) do
+            if #k >= slen then
+                val = slist[k:sub(1, slen)]
+                if val ~= nil then
+                    return val
+                end
+            end
+        end
+    end
+end
+
 -- Process functions for all data types
-function osm2pgsql.process_node(object)
+function module.process_node(object)
 
     local function geom_func(o)
         return o:as_point()
     end
 
-    process_tags(Place.new(object, geom_func))
+    module.process_tags(Place.new(object, geom_func))
 end
 
-function osm2pgsql.process_way(object)
+function module.process_way(object)
 
     local function geom_func(o)
         local geom = o:as_polygon()
@@ -298,30 +394,24 @@ function osm2pgsql.process_way(object)
         return geom
     end
 
-    process_tags(Place.new(object, geom_func))
+    module.process_tags(Place.new(object, geom_func))
 end
 
-function relation_as_multipolygon(o)
+function module.process_relation(object)
-    return o:as_multipolygon()
+    local geom_func = module.RELATION_TYPES[object.tags.type]
-end
-
-function relation_as_multiline(o)
-    return o:as_multilinestring():line_merge()
-end
-
-function osm2pgsql.process_relation(object)
-    local geom_func = RELATION_TYPES[object.tags.type]
 
     if geom_func ~= nil then
-        process_tags(Place.new(object, geom_func))
+        module.process_tags(Place.new(object, geom_func))
     end
 end
 
-function process_tags(o)
+-- The process functions are used by default by osm2pgsql.
-    local fallback
+osm2pgsql.process_node = module.process_node
+osm2pgsql.process_way = module.process_way
+osm2pgsql.process_relation = module.process_relation
 
-    o:delete{match = PRE_DELETE}
+function module.process_tags(o)
-    o:grab_extratags{match = PRE_EXTRAS}
+    o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS}
 
     -- Exception for boundary/place double tagging
     if o.object.tags.boundary == 'administrative' then
@@ -330,54 +420,91 @@ function process_tags(o)
         end}
     end
 
+    -- name keys
+    local fallback = o:grab_name_parts{groups=NAMES}
+
     -- address keys
-    o:grab_address{match=COUNTRY_TAGS, out_key='country'}
+    if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then
+        fallback = {'place', 'house', 'always'}
+    end
     if o.address.country ~= nil and #o.address.country ~= 2 then
         o.address['country'] = nil
     end
-    if o:grab_name{match=HOUSENAME_TAGS} > 0 then
+    if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then
-        fallback = {'place', 'house'}
+        fallback = {'place', 'postcode', 'always'}
-    end
-    if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then
-        fallback = {'place', 'house'}
-    end
-    if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then
-        fallback = {'place', 'postcode'}
     end
 
-    local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0
+    if o.address.interpolation ~= nil then
-
-    o:grab_address{match=ADDRESS_TAGS}
-
-    if is_interpolation then
         o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
         return
     end
 
-    -- name keys
+    o:clean{delete = POST_DELETE}
-    o:grab_name{match = NAMES}
-    o:grab_name{match = REFS, include_on_name = false}
-
-    o:delete{match = POST_DELETE}
-    o:grab_extratags{match = POST_EXTRAS}
 
     -- collect main keys
-    local num_mains = 0
+    for k, v in pairs(o.object.tags) do
-    for k, v in pairs(o:tags()) do
+        local ktype = MAIN_KEYS[k]
-        num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
+        if ktype == 'fallback' then
+            if o.has_name then
+                fallback = {k, v, 'named'}
+            end
+        elseif ktype ~= nil then
+            o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
+        end
     end
 
-    if num_mains == 0 then
+    if fallback ~= nil and o.num_entries == 0 then
-        for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do
+        o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS)
-            if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then
+    end
-                return
+end
-            end
-        end
 
-        if fallback ~= nil then
+--------- Convenience functions for simple style configuration -----------------
-            o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS)
+
+
+function module.set_prefilters(data)
+    PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
+    PRE_EXTRAS = module.tag_match{keys = data.extratag_keys,
+                                  tags = data.extratag_tags}
+end
+
+function module.set_main_tags(data)
+    MAIN_KEYS = data
+end
+
+function module.set_name_tags(data)
+    NAMES = module.tag_group(data)
+end
+
+function module.set_address_tags(data)
+    if data.postcode_fallback ~= nil then
+        POSTCODE_FALLBACK = data.postcode_fallback
+        data.postcode_fallback = nil
+    end
+
+    ADDRESS_TAGS = module.tag_group(data)
+end
+
+function module.set_unused_handling(data)
+    if data.extra_keys == nil and data.extra_tags == nil then
+        POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags}
+        SAVE_EXTRA_MAINS = function() return true end
+    elseif data.delete_keys == nil and data.delete_tags == nil then
+        POST_DELETE = nil
+        SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags}
+    else
+        error("unused handler can have only 'extra_keys' or 'delete_keys' set.")
+    end
+end
+
+function set_relation_types(data)
+    module.RELATION_TYPES = {}
+    for k, v in data do
+        if v == 'multipolygon' then
+            module.RELATION_TYPES[k] = module.relation_as_multipolygon
+        elseif v == 'multiline' then
+            module.RELATION_TYPES[k] = module.relation_as_multiline
         end
     end
 end
 
-
+return module

settings/icu_tokenizer.yaml

@@ -24,6 +24,7 @@ transliteration:
     - ":: lower ()"
     - "[^a-z0-9[:Space:]] >"
     - ":: NFC ()"
+    - "[:Space:]+ > ' '"
 sanitizers:
     - step: clean-housenumbers
       filter-kind:
@@ -37,6 +38,7 @@ sanitizers:
       default-pattern: "[A-Z0-9- ]{3,12}"
     - step: clean-tiger-tags
     - step: split-name-list
+      delimiters: ;
     - step: strip-brace-terms
     - step: tag-analyzer-by-language
       filter-kind: [".*name.*"]

settings/import-address.lua

@@ -0,0 +1,67 @@
+flex = require('flex-base')
+
+flex.set_main_tags{
+    highway = {'always',
+               street_lamp = 'named',
+               traffic_signals = 'named',
+               service = 'named',
+               cycleway = 'named',
+               path = 'named',
+               footway = 'named',
+               steps = 'named',
+               bridleway = 'named',
+               track = 'named',
+               motorway_link = 'named',
+               trunk_link = 'named',
+               primary_link = 'named',
+               secondary_link = 'named',
+               tertiary_link = 'named'},
+    boundary = {administrative = 'named',
+                postal_code = 'named'},
+    landuse = 'fallback',
+    place = 'always'
+}
+
+flex.set_prefilters{delete_keys = {'building', 'source',
+                                   'source', '*source', 'type',
+                                   'is_in:postcode', '*:wikidata', '*:wikipedia',
+                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                                   'name:etymology', 'name:signed', 'name:botanical',
+                                   'addr:street:name', 'addr:street:type'},
+                    delete_tags = {highway = {'no', 'turning_circle', 'mini_roundabout',
+                                              'noexit', 'crossing', 'give_way', 'stop'},
+                                   landuse = {'cemetry', 'no'},
+                                   boundary = {'place'}},
+                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
+                   }
+
+flex.set_name_tags{main = {'name', 'name:*',
+                          'int_name', 'int_name:*',
+                          'nat_name', 'nat_name:*',
+                          'reg_name', 'reg_name:*',
+                          'loc_name', 'loc_name:*',
+                          'old_name', 'old_name:*',
+                          'alt_name', 'alt_name:*', 'alt_name_*',
+                          'official_name', 'official_name:*',
+                          'place_name', 'place_name:*',
+                          'short_name', 'short_name:*'},
+                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
+                            'loc_ref', 'old_ref',
+                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'},
+                   house = {'addr:housename'}
+                  }
+
+flex.set_address_tags{main = {'addr:housenumber',
+                              'addr:conscriptionnumber',
+                              'addr:streetnumber'},
+                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
+                      postcode = {'postal_code', 'postcode', 'addr:postcode',
+                                  'tiger:zip_left', 'tiger:zip_right'},
+                      country = {'country_code', 'ISO3166-1',
+                                 'addr:country_code', 'is_in:country_code',
+                                 'addr:country', 'is_in:country'},
+                      interpolation = {'addr:interpolation'}
+                     }
+
+
+flex.set_unused_handling{extra_keys = {'place'}}

settings/import-admin.lua

@@ -0,0 +1,44 @@
+local flex = require('flex-base')
+
+flex.set_main_tags{
+    boundary = {administrative = 'named'},
+    landuse = 'fallback',
+    place = 'always'
+}
+
+flex.set_prefilters{delete_keys = {'building', 'source', 'highway',
+                                   'addr:housenumber', 'addr:street', 'addr:city',
+                                   'source', '*source', 'type',
+                                   'is_in:postcode', '*:wikidata', '*:wikipedia',
+                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                                   'name:etymology', 'name:signed', 'name:botanical',
+                                   'addr:street:name', 'addr:street:type'},
+                    delete_tags = {landuse = {'cemetry', 'no'},
+                                   boundary = {'place'}},
+                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'}
+                   }
+
+flex.set_name_tags{main = {'name', 'name:*',
+                          'int_name', 'int_name:*',
+                          'nat_name', 'nat_name:*',
+                          'reg_name', 'reg_name:*',
+                          'loc_name', 'loc_name:*',
+                          'old_name', 'old_name:*',
+                          'alt_name', 'alt_name:*', 'alt_name_*',
+                          'official_name', 'official_name:*',
+                          'place_name', 'place_name:*',
+                          'short_name', 'short_name:*'},
+                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
+                            'loc_ref', 'old_ref',
+                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}
+                  }
+
+flex.set_address_tags{extra = {'addr:*', 'is_in:*'},
+                      postcode = {'postal_code', 'postcode', 'addr:postcode'},
+                      country = {'country_code', 'ISO3166-1',
+                                 'addr:country_code', 'is_in:country_code',
+                                 'addr:country', 'is_in:country'},
+                      postcode_fallback = false
+                     }
+
+flex.set_unused_handling{extra_keys = {'place'}}

settings/import-extratags.lua

@@ -1,13 +1,9 @@
-require('flex-base')
+flex = require('flex-base')
 
-RELATION_TYPES = {
+flex.set_main_tags{
-    multipolygon = relation_as_multipolygon,
+    building = 'fallback',
-    boundary = relation_as_multipolygon,
-    waterway = relation_as_multiline
-}
-
-MAIN_KEYS = {
     emergency = 'always',
+    healthcare = 'fallback',
     historic = 'always',
     military = 'always',
     natural = 'named',
@@ -36,6 +32,8 @@ MAIN_KEYS = {
     amenity = 'always',
     club = 'always',
     craft = 'always',
+    junction = 'fallback',
+    landuse = 'fallback',
     leisure = 'always',
     office = 'always',
     mountain_pass = 'always',
@@ -47,55 +45,43 @@ MAIN_KEYS = {
     place = 'always'
 }
 
-MAIN_FALLBACK_KEYS = {
+flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attribution',
-    building = 'named',
+                                   'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
-    landuse = 'named',
+                                   'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
-    junction = 'named',
+                                   'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
-    healthcare = 'named'
+                                   'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
-}
+                                   'ref:linz:*', 'is_in:postcode'},
-
+                    delete_tags = {emergency = {'yes', 'no', 'fire_hydrant'},
-
+                                   historic = {'yes', 'no'},
-PRE_DELETE = tag_match{keys = {'note', 'note:*', 'source', 'source*', 'attribution',
+                                   military = {'yes', 'no'},
-                               'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
+                                   natural = {'yes', 'no', 'coastline'},
-                               'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
+                                   highway = {'no', 'turning_circle', 'mini_roundabout',
-                               'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
+                                              'noexit', 'crossing', 'give_way', 'stop'},
-                               'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
+                                   railway = {'level_crossing', 'no', 'rail'},
-                               'ref:linz:*', 'is_in:postcode'},
+                                   man_made = {'survey_point', 'cutline'},
-                       tags = {emergency = {'yes', 'no', 'fire_hydrant'},
+                                   aerialway = {'pylon', 'no'},
-                               historic = {'yes', 'no'},
+                                   aeroway = {'no'},
-                               military = {'yes', 'no'},
+                                   amenity = {'no'},
-                               natural = {'yes', 'no', 'coastline'},
+                                   club = {'no'},
-                               highway = {'no', 'turning_circle', 'mini_roundabout',
+                                   craft = {'no'},
-                                          'noexit', 'crossing', 'give_way', 'stop'},
+                                   leisure = {'no'},
-                               railway = {'level_crossing', 'no', 'rail'},
+                                   office = {'no'},
-                               man_made = {'survey_point', 'cutline'},
+                                   mountain_pass = {'no'},
-                               aerialway = {'pylon', 'no'},
+                                   shop = {'no'},
-                               aeroway = {'no'},
+                                   tourism = {'yes', 'no'},
-                               amenity = {'no'},
+                                   bridge = {'no'},
-                               club = {'no'},
+                                   tunnel = {'no'},
-                               craft = {'no'},
+                                   waterway = {'riverbank'},
-                               leisure = {'no'},
+                                   building = {'no'},
-                               office = {'no'},
+                                   boundary = {'place'}},
-                               mountain_pass = {'no'},
+                    extratag_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
-                               shop = {'no'},
-                               tourism = {'yes', 'no'},
-                               bridge = {'no'},
-                               tunnel = {'no'},
-                               waterway = {'riverbank'},
-                               building = {'no'},
-                               boundary = {'place'}}
-                      }
-
-POST_DELETE = tag_match{keys = {'tiger:*'}}
-
-PRE_EXTRAS = tag_match{keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
                                'name:etymology', 'name:signed', 'name:botanical',
                                'wikidata', '*:wikidata',
+                               '*:wikipedia', 'brand:wikipedia:*',
                                'addr:street:name', 'addr:street:type'}
-                      }
+                   }
 
-
+flex.set_name_tags{main = {'name', 'name:*',
-NAMES = tag_match{keys = {'name', 'name:*',
                           'int_name', 'int_name:*',
                           'nat_name', 'nat_name:*',
                           'reg_name', 'reg_name:*',
@@ -104,26 +90,24 @@ NAMES = tag_match{keys = {'name', 'name:*',
                           'alt_name', 'alt_name:*', 'alt_name_*',
                           'official_name', 'official_name:*',
                           'place_name', 'place_name:*',
-                          'short_name', 'short_name:*', 'brand'}}
+                          'short_name', 'short_name:*', 'brand'},
+                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
+                            'loc_ref', 'old_ref',
+                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'},
+                   house = {'addr:housename'}
+                  }
 
-REFS = tag_match{keys = {'ref', 'int_ref', 'nat_ref', 'reg_ref', 'loc_ref', 'old_ref',
+flex.set_address_tags{main = {'addr:housenumber',
-                         'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}}
+                              'addr:conscriptionnumber',
-
+                              'addr:streetnumber'},
-POSTCODES = tag_match{keys = {'postal_code', 'postcode', 'addr:postcode',
+                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
-                              'tiger:zip_left', 'tiger:zip_right'}}
+                      postcode = {'postal_code', 'postcode', 'addr:postcode',
-
+                                  'tiger:zip_left', 'tiger:zip_right'},
-COUNTRY_TAGS = tag_match{keys = {'country_code', 'ISO3166-1',
+                      country = {'country_code', 'ISO3166-1',
                                  'addr:country_code', 'is_in:country_code',
-                                 'addr:country', 'is_in:country'}}
+                                 'addr:country', 'is_in:country'},
+                      interpolation = {'addr:interpolation'}
+                     }
 
-HOUSENAME_TAGS = tag_match{keys = {'addr:housename'}}
-
-HOUSENUMBER_TAGS = tag_match{keys = {'addr:housenumber', 'addr:conscriptionnumber',
-                                     'addr:streetnumber'}}
-
-INTERPOLATION_TAGS = tag_match{keys = {'addr:interpolation'}}
-
-ADDRESS_TAGS = tag_match{keys = {'addr:*', 'is_in:*', 'tiger:county'}}
-
-SAVE_EXTRA_MAINS = true
 
+flex.set_unused_handling{delete_keys = {'tiger:*'}}

settings/import-full.lua

@@ -0,0 +1,113 @@
+flex = require('flex-base')
+
+flex.set_main_tags{
+    building = 'fallback',
+    emergency = 'always',
+    healthcare = 'fallback',
+    historic = 'always',
+    military = 'always',
+    natural = 'named',
+    landuse = 'named',
+    highway = {'always',
+               street_lamp = 'named',
+               traffic_signals = 'named',
+               service = 'named',
+               cycleway = 'named',
+               path = 'named',
+               footway = 'named',
+               steps = 'named',
+               bridleway = 'named',
+               track = 'named',
+               motorway_link = 'named',
+               trunk_link = 'named',
+               primary_link = 'named',
+               secondary_link = 'named',
+               tertiary_link = 'named'},
+    railway = 'named',
+    man_made = 'always',
+    aerialway = 'always',
+    boundary = {'named',
+                postal_code = 'named'},
+    aeroway = 'always',
+    amenity = 'always',
+    club = 'always',
+    craft = 'always',
+    junction = 'fallback',
+    landuse = 'fallback',
+    leisure = 'always',
+    office = 'always',
+    mountain_pass = 'always',
+    shop = 'always',
+    tourism = 'always',
+    bridge = 'named_with_key',
+    tunnel = 'named_with_key',
+    waterway = 'named',
+    place = 'always'
+}
+
+flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attribution',
+                                   'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
+                                   'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
+                                   'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
+                                   'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
+                                   'ref:linz:*', 'is_in:postcode',
+                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                                   'name:etymology', 'name:signed', 'name:botanical',
+                                   '*:wikidata', '*:wikipedia', 'brand:wikipedia:*',
+                                   'addr:street:name', 'addr:street:type'},
+                    delete_tags = {emergency = {'yes', 'no', 'fire_hydrant'},
+                                   historic = {'yes', 'no'},
+                                   military = {'yes', 'no'},
+                                   natural = {'yes', 'no', 'coastline'},
+                                   highway = {'no', 'turning_circle', 'mini_roundabout',
+                                              'noexit', 'crossing', 'give_way', 'stop'},
+                                   railway = {'level_crossing', 'no', 'rail'},
+                                   man_made = {'survey_point', 'cutline'},
+                                   aerialway = {'pylon', 'no'},
+                                   aeroway = {'no'},
+                                   amenity = {'no'},
+                                   club = {'no'},
+                                   craft = {'no'},
+                                   leisure = {'no'},
+                                   office = {'no'},
+                                   mountain_pass = {'no'},
+                                   shop = {'no'},
+                                   tourism = {'yes', 'no'},
+                                   bridge = {'no'},
+                                   tunnel = {'no'},
+                                   waterway = {'riverbank'},
+                                   building = {'no'},
+                                   boundary = {'place'}},
+                    extra_keys = {'wikidata', 'wikipedia', 'wikipedia:*'}
+                   }
+
+flex.set_name_tags{main = {'name', 'name:*',
+                          'int_name', 'int_name:*',
+                          'nat_name', 'nat_name:*',
+                          'reg_name', 'reg_name:*',
+                          'loc_name', 'loc_name:*',
+                          'old_name', 'old_name:*',
+                          'alt_name', 'alt_name:*', 'alt_name_*',
+                          'official_name', 'official_name:*',
+                          'place_name', 'place_name:*',
+                          'short_name', 'short_name:*', 'brand'},
+                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
+                            'loc_ref', 'old_ref',
+                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'},
+                   house = {'addr:housename'}
+                  }
+
+flex.set_address_tags{main = {'addr:housenumber',
+                              'addr:conscriptionnumber',
+                              'addr:streetnumber'},
+                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
+                      postcode = {'postal_code', 'postcode', 'addr:postcode',
+                                  'tiger:zip_left', 'tiger:zip_right'},
+                      country = {'country_code', 'ISO3166-1',
+                                 'addr:country_code', 'is_in:country_code',
+                                 'addr:country', 'is_in:country'},
+                      interpolation = {'addr:interpolation'}
+                     }
+
+
+flex.set_unused_handling{extra_keys = {'place'}}

settings/import-street.lua

@@ -0,0 +1,66 @@
+flex = require('flex-base')
+
+flex.set_main_tags{
+    highway = {'always',
+               street_lamp = 'named',
+               traffic_signals = 'named',
+               service = 'named',
+               cycleway = 'named',
+               path = 'named',
+               footway = 'named',
+               steps = 'named',
+               bridleway = 'named',
+               track = 'named',
+               motorway_link = 'named',
+               trunk_link = 'named',
+               primary_link = 'named',
+               secondary_link = 'named',
+               tertiary_link = 'named'},
+    boundary = {administrative = 'named'},
+    landuse = 'fallback',
+    place = 'always'
+}
+
+flex.set_prefilters{delete_keys = {'building', 'source',
+                                   'addr:housenumber', 'addr:street',
+                                   'source', '*source', 'type',
+                                   'is_in:postcode', '*:wikidata', '*:wikipedia',
+                                   '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                                   'name:etymology', 'name:signed', 'name:botanical',
+                                   'addr:street:name', 'addr:street:type'},
+                    delete_tags = {highway = {'no', 'turning_circle', 'mini_roundabout',
+                                              'noexit', 'crossing', 'give_way', 'stop'},
+                                   landuse = {'cemetry', 'no'},
+                                   boundary = {'place'}},
+                    extratag_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'}
+                   }
+
+flex.set_name_tags{main = {'name', 'name:*',
+                          'int_name', 'int_name:*',
+                          'nat_name', 'nat_name:*',
+                          'reg_name', 'reg_name:*',
+                          'loc_name', 'loc_name:*',
+                          'old_name', 'old_name:*',
+                          'alt_name', 'alt_name:*', 'alt_name_*',
+                          'official_name', 'official_name:*',
+                          'place_name', 'place_name:*',
+                          'short_name', 'short_name:*'},
+                   extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref',
+                            'loc_ref', 'old_ref',
+                            'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}
+                  }
+
+flex.set_address_tags{main = {'addr:housenumber',
+                              'addr:conscriptionnumber',
+                              'addr:streetnumber'},
+                      extra = {'addr:*', 'is_in:*', 'tiger:county'},
+                      postcode = {'postal_code', 'postcode', 'addr:postcode',
+                                  'tiger:zip_left', 'tiger:zip_right'},
+                      country = {'country_code', 'ISO3166-1',
+                                 'addr:country_code', 'is_in:country_code',
+                                 'addr:country', 'is_in:country'},
+                      interpolation = {'addr:interpolation'},
+                      postcode_fallback = false
+                     }
+
+flex.set_unused_handling{extra_keys = {'place'}}

test/Makefile

@@ -1,12 +1,9 @@
-all: bdd php
+all: bdd php python
 no-test-db: bdd-no-test-db php
 
 bdd:
 	cd bdd && behave -DREMOVE_TEMPLATE=1
 
-icu:
-	cd bdd && behave -DREMOVE_TEMPLATE=1 -DTOKENIZER=icu
-
 php:
 	cd php && phpunit ./
 
@@ -14,4 +11,4 @@ python:
 	pytest python
 
 
-.PHONY: bdd php no-test-db
+.PHONY: bdd php no-test-db python

test/bdd/db/import/rank_computation.feature

@@ -255,3 +255,15 @@ Feature: Rank assignment
          | W1     | R10     | True      | 18                  |
          | W1     | R2      | True      | 16                  |
          | W1     | N9      | False     | 18                  |
+
+
+    Scenario: POI nodes with place tags
+        Given the places
+          | osm | class   | type       | name | extratags       |
+          | N23 | amenity | playground | AB   | "place": "city" |
+          | N23 | place   | city       | AB   | "amenity": "playground" |
+        When importing
+        Then placex contains exactly
+          | object      | rank_search | rank_address |
+          | N23:amenity | 30          | 30           |
+          | N23:place   | 16          | 16           |

test/bdd/db/import/search_name.feature

@@ -2,21 +2,26 @@
 Feature: Creation of search terms
     Tests that search_name table is filled correctly
 
-    Scenario Outline: Comma- and semicolon separated names appear as full names
+    Scenario: Semicolon-separated names appear as separate full names
         Given the places
          | osm | class   | type | name+alt_name |
-         | N1  | place   | city | New York<sep>Big Apple |
+         | N1  | place   | city | New York; Big Apple |
         When importing
         Then search_name contains
          | object | name_vector |
          | N1     | #New York, #Big Apple |
 
-    Examples:
+    @fail-legacy
-         | sep |
+    Scenario: Comma-separated names appear as a single full name
-         | ,   |
+        Given the places
-         | ;   |
+         | osm | class   | type | name+alt_name |
+         | N1  | place   | city | New York, Big Apple |
+        When importing
+        Then search_name contains
+         | object | name_vector |
+         | N1     | #New York Big Apple |
 
-    Scenario Outline: Name parts before brackets appear as full names
+    Scenario: Name parts before brackets appear as full names
         Given the places
          | osm | class   | type | name+name |
          | N1  | place   | city | Halle (Saale) |

test/bdd/osm2pgsql/import/tags.feature

@@ -122,10 +122,10 @@ Feature: Tag evaluation
             n8003 Tshop=shoes,name:source=survey
             """
         Then place contains exactly
-            | object | class | extratags    |
+            | object | class | name | extratags    |
-            | N8001  | shop  | 'xx': 'yy'   |
+            | N8001  | shop  |  -   | 'xx': 'yy'   |
-            | N8002  | shop  | 'ele': '234' |
+            | N8002  | shop  |  -   | 'ele': '234' |
-            | N8003  | shop  | -            |
+            | N8003  | shop  |  -   | -            |
 
 
     Scenario: Admin levels

test/php/Nominatim/DebugTest.php

@@ -37,14 +37,14 @@ class DebugTest extends \PHPUnit\Framework\TestCase
 <pre><b>Var1:</b>  <i>True</i></pre>
 <pre><b>Var2:</b>  <i>False</i></pre>
 <pre><b>Var3:</b>  0</pre>
-<pre><b>Var4:</b>  'String'</pre>
+<pre><b>Var4:</b>  &#039;String&#039;</pre>
-<pre><b>Var5:</b>  0 => 'one'
+<pre><b>Var5:</b>  0 => &#039;one&#039;
-       1 => 'two'
+       1 => &#039;two&#039;
-       2 => 'three'</pre>
+       2 => &#039;three&#039;</pre>
-<pre><b>Var6:</b>  'key' => 'value'
+<pre><b>Var6:</b>  &#039;key&#039; => &#039;value&#039;
-       'key2' => 'value2'</pre>
+       &#039;key2&#039; => &#039;value2&#039;</pre>
 <pre><b>Var7:</b>  me as string</pre>
-<pre><b>Var8:</b>  'value', 'value2'</pre>
+<pre><b>Var8:</b>  &#039;value&#039;, &#039;value2&#039;</pre>
 
 EOT
         );
@@ -64,10 +64,10 @@ EOT
     public function testDebugArray()
     {
         $this->expectOutputString(<<<EOT
-<pre><b>Arr0:</b>  'null'</pre>
+<pre><b>Arr0:</b>  &#039;null&#039;</pre>
-<pre><b>Arr1:</b>  'key1' => 'val1'
+<pre><b>Arr1:</b>  &#039;key1&#039; => &#039;val1&#039;
-       'key2' => 'val2'
+       &#039;key2&#039; => &#039;val2&#039;
-       'key3' => 'val3'</pre>
+       &#039;key3&#039; => &#039;val3&#039;</pre>
 
 EOT
         );
@@ -93,12 +93,12 @@ EOT
     <th><small>1</small></th>
   </tr>
   <tr>
-    <td><pre>'one'</pre></td>
+    <td><pre>&#039;one&#039;</pre></td>
-    <td><pre>'two'</pre></td>
+    <td><pre>&#039;two&#039;</pre></td>
   </tr>
   <tr>
-    <td><pre>'three'</pre></td>
+    <td><pre>&#039;three&#039;</pre></td>
-    <td><pre>'four'</pre></td>
+    <td><pre>&#039;four&#039;</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -109,9 +109,9 @@ EOT
     <th><small>key3</small></th>
   </tr>
   <tr>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
 </table>
 
@@ -147,18 +147,18 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'one'</pre></td>
+    <td><pre>&#039;one&#039;</pre></td>
-    <td><pre>'two'</pre></td>
+    <td><pre>&#039;two&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group2</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -171,15 +171,15 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
 </table>
 

test/python/tools/test_refresh.py

@@ -48,7 +48,7 @@ def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, r
 def test_recompute_importance(placex_table, table_factory, temp_db_conn, temp_db_cursor):
     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE,
                                               country_code varchar(2),
-                                              osm_type varchar(1), osm_id BIGINT,
+                                              rank_search SMALLINT,
                                               centroid GEOMETRY,
                                               OUT importance FLOAT,
                                               OUT wikipedia TEXT)

vagrant/Install-on-Ubuntu-18.sh

@@ -115,7 +115,7 @@ fi                                 #DOCS:
 # download the country grid:
 
 if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
-    wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
 fi                                 #DOCS:
 
 # The code must be built in a separate directory. Create this directory,

vagrant/Install-on-Ubuntu-20.sh

@@ -109,7 +109,7 @@ fi                                 #DOCS:
 # download the country grid:
 
 if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
-    wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
 fi                                 #DOCS:
 
 # The code must be built in a separate directory. Create this directory,

vagrant/Install-on-Ubuntu-22.sh

@@ -109,7 +109,7 @@ fi                                 #DOCS:
 # download the country grid:
 
 if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
-    wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+    wget -O data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
 fi                                 #DOCS:
 
 # The code must be built in a separate directory. Create this directory,