prepare release 4.2.1

Add migration for reorganization of pending indexes

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: lonvia
+custom: "https://nominatim.org/funding/"

.github/actions/build-nominatim/action.yml

@@ -9,6 +9,10 @@ inputs:
         description: 'Additional options to hand to cmake'
         required: false
         default: ''
+    lua:
+        description: 'Version of Lua to use'
+        required: false
+        default: '5.3'
 
 runs:
     using: "composite"
@@ -21,7 +25,7 @@ runs:
           shell: bash
         - name: Install prerequisites
           run: |
-            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev
+            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION}
             if [ "x$UBUNTUVER" == "x18" ]; then
                 pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 datrie
             else
@@ -31,6 +35,7 @@ runs:
           env:
             UBUNTUVER: ${{ inputs.ubuntu }}
             CMAKE_ARGS: ${{ inputs.cmake-args }}
+            LUA_VERSION: ${{ inputs.lua }}
 
         - name: Configure
           run: mkdir build && cd build && cmake $CMAKE_ARGS ../Nominatim

.github/actions/setup-postgresql/action.yml

@@ -15,7 +15,9 @@ runs:
         - name: Remove existing PostgreSQL
           run: |
               sudo apt-get purge -yq postgresql*
-              sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+              sudo apt install curl ca-certificates gnupg
+              curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
+              sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
               sudo apt-get update -qq
 
           shell: bash

.github/workflows/ci-tests.yml

@@ -50,7 +50,7 @@ jobs:
                       pytest: py.test-3
                       php: 7.4
                     - ubuntu: 22
-                      postgresql: 14
+                      postgresql: 15
                       postgis: 3
                       pytest: py.test-3
                       php: 8.1
@@ -265,6 +265,10 @@ jobs:
               run: nominatim --version
               working-directory: /home/nominatim/nominatim-project
 
+            - name: Collect host OS information
+              run: nominatim admin --collect-os-info
+              working-directory: /home/nominatim/nominatim-project
+              
             - name: Import
               run: nominatim import --osm-file ../test.pbf
               working-directory: /home/nominatim/nominatim-project

.pylintrc

@@ -15,4 +15,4 @@ ignored-classes=NominatimArgs,closing
 #   typed Python is enabled. See also https://github.com/PyCQA/pylint/issues/5273
 disable=too-few-public-methods,duplicate-code,too-many-ancestors,bad-option-value,no-self-use,not-context-manager
 
-good-names=i,x,y,fd,db,cc
+good-names=i,x,y,m,fd,db,cc

CMakeLists.txt

@@ -19,8 +19,8 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 4)
-set(NOMINATIM_VERSION_MINOR 1)
+set(NOMINATIM_VERSION_MINOR 2)
-set(NOMINATIM_VERSION_PATCH 2)
+set(NOMINATIM_VERSION_PATCH 1)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 
@@ -63,7 +63,6 @@ if (BUILD_IMPORTER AND BUILD_OSM2PGSQL)
     endif()
     set(BUILD_TESTS_SAVED "${BUILD_TESTS}")
     set(BUILD_TESTS off)
-    set(WITH_LUA off CACHE BOOL "")
     add_subdirectory(osm2pgsql)
     set(BUILD_TESTS ${BUILD_TESTS_SAVED})
 endif()

CONTRIBUTING.md

@@ -64,3 +64,39 @@ Before submitting a pull request make sure that the tests pass:
   cd build
   make test
 ```
+
+## Releases
+
+Nominatim follows semantic versioning. Major releases are done for large changes
+that require (or at least strongly recommend) a reimport of the databases.
+Minor releases can usually be applied to exisiting databases Patch releases
+contain bug fixes only and are released from a separate branch where the
+relevant changes are cherry-picked from the master branch.
+
+Checklist for releases:
+
+* [ ] increase version in `nominatim/version.py` and CMakeLists.txt
+* [ ] update `ChangeLog` (copy information from patch releases from release branch)
+* [ ] complete `docs/admin/Migration.md`
+* [ ] update EOL dates in `SECURITY.md`
+* [ ] commit and make sure CI tests pass
+* [ ] test migration
+  * download, build and import previous version
+  * migrate using master version
+  * run updates using master version
+* [ ] prepare tarball:
+  * `git clone --recursive https://github.com/osm-search/Nominatim` (switch to right branch!)
+  * `rm -r .git* osm2pgsql/.git*`
+  * copy country data into `data/`
+  * add version to base directory and package
+* [ ] upload tarball to https://nominatim.org
+* [ ] prepare documentation
+  * check out new docs branch
+  * change git checkout instructions to tarball download instructions or adapt version on existing ones
+  * build documentation and copy to https://github.com/osm-search/nominatim-org-site
+  * add new version to history
+* [ ] check release tarball
+  * download tarball as per new documentation instructions
+  * compile and import Nominatim
+  * run `nominatim --version` to confirm correct version
+* [ ] tag new release and add a release on github.com

ChangeLog

@@ -1,11 +1,19 @@
-4.1.2
+4.2.1
 
  * fix XSS vulnerability in debug view
 
-4.1.1
+4.2.0
 
+ * add experimental support for osm2pgsql flex style
+ * introduce secondary importance value to be retrieved from a raster data file
+   (currently still unused, to replace address importance, thanks to @tareqpi)
+ * add new report tool `nominatim admin --collect-os-info`
+   (thanks @micahcochran, @tareqpi)
+ * reorganise index to improve lookup performance and size
+ * run index creation after import in parallel
+ * run ANALYZE more selectively to speed up continuation of indexing
  * fix crash on update when addr:interpolation receives an illegal value
- * fix minimum number of retrived results to be at least 10
+ * fix minimum number of retrieved results to be at least 10
  * fix search for combinations of special term + name (e.g Hotel Bellevue)
  * do not return interpolations without a parent street on reverse search
  * improve invalidation of linked places on updates
@@ -13,8 +21,13 @@
  * make sure socket timeouts are respected during replication
    (working around a bug in some versions of pyosmium)
  * update bundled osm2pgsql to 1.7.1
+ * add support for PostgreSQL 15
  * typing fixes to work with latest type annotations from typeshed
- * smaller improvements to documention (thanks to @mausch)
+ * smaller improvements to documentation (thanks to @mausch)
+
+4.1.1
+
+ * fix XSS vulnerability in debug view
 
 4.1.0
 

SECURITY.md

@@ -9,10 +9,11 @@ versions.
 
 | Version | End of support for security updates |
 | ------- | ----------------------------------- |
+| 4.2.x   | 2024-11-24                          |
+| 4.1.x   | 2024-08-05                          |
 | 4.0.x   | 2023-11-02                          |
 | 3.7.x   | 2023-04-05                          |
 | 3.6.x   | 2022-12-12                          |
-| 3.5.x   | 2022-06-05                          |
 
 ## Reporting a Vulnerability
 

docs/admin/Import.md

@@ -79,10 +79,10 @@ This data is available as a binary download. Put it into your project directory:
 The file is about 400MB and adds around 4GB to the Nominatim database.
 
 !!! tip
-    If you forgot to download the wikipedia rankings, you can also add
+    If you forgot to download the wikipedia rankings, then you can
-    importances after the import. Download the files, then run
+    also add importances after the import. Download the SQL files, then
-    `nominatim refresh --wiki-data --importance`. Updating importances for
+    run `nominatim refresh --wiki-data --importance`. Updating
-    a planet can take a couple of hours.
+    importances for a planet will take a couple of hours.
 
 ### External postcodes
 
@@ -139,7 +139,7 @@ import. So this option is particularly interesting if you plan to transfer the
 database or reuse the space later.
 
 !!! warning
-    The datastructure for updates are also required when adding additional data
+    The data structure for updates are also required when adding additional data
     after the import, for example [TIGER housenumber data](../customize/Tiger.md).
     If you plan to use those, you must not use the `--no-updates` parameter.
     Do a normal import, add the external data and once you are done with

docs/api/Status.md

@@ -57,10 +57,11 @@ code and message, e.g.
 
 Possible status codes are
 
-   |     | message              | notes                                             |
+|     | message                        | notes                                                             |
-   |-----|----------------------|---------------------------------------------------|
+| --- | ------------------------------ | ----------------------------------------------------------------- |
-   | 700 | "No database"        | connection failed                                 |
+| 700 | "No database"                  | connection failed                                                 |
-   | 701 | "Module failed"      | database could not load nominatim.so              |
+| 701 | "Module failed"                | database could not load nominatim.so                              |
-   | 702 | "Module call failed" | nominatim.so loaded but calling a function failed |
+| 702 | "Module call failed"           | nominatim.so loaded but calling a function failed                 |
-   | 703 | "Query failed"       | test query against a database table failed        |
+| 703 | "Query failed"                 | test query against a database table failed                        |
-   | 704 | "No value"           | test query worked but returned no results         |
+| 704 | "No value"                     | test query worked but returned no results                         |
+| 705 | "Import date is not available" | No import dates were returned (enabling replication can fix this) |

docs/customize/Importance.md

@@ -0,0 +1,49 @@
+## Importance
+
+Search requests can yield multiple results which match equally well with
+the original query. In such case Nominatim needs to order the results
+according to a different criterion: importance. This is a measure for how
+likely it is that a user will search for a given place. This section explains
+the sources Nominatim uses for computing importance of a place and how to
+customize them.
+
+### How importance is computed
+
+The main value for importance is derived from page ranking values for Wikipedia
+pages for a place. For places that do not have their own
+Wikipedia page, a formula is used that derives a static importance from the
+places [search rank](../customize/Ranking#search-rank).
+
+In a second step, a secondary importance value is added which is meant to
+represent how well-known the general area is where the place is located. It
+functions as a tie-breaker between places with very similar primary
+importance values.
+
+nominatim.org has preprocessed importance tables for the
+[primary Wikipedia rankings](https://nominatim.org/data/wikimedia-importance.sql.gz)
+and for a secondary importance based on the number of tile views on openstreetmap.org.
+
+### Customizing secondary importance
+
+The secondary importance is implemented as a simple
+[Postgis raster](https://postgis.net/docs/raster.html) table, where Nominatim
+looks up the value for the coordinates of the centroid of a place. You can
+provide your own secondary importance raster in form of an SQL file named
+`secondary_importance.sql.gz` in your project directory.
+
+The SQL file needs to drop and (re)create a table `secondary_importance` which
+must as a minimum contain a column `rast` of type `raster`. The raster must
+be in EPSG:4326 and contain 16bit unsigned ints
+(`raster_constraint_pixel_types(rast) = '{16BUI}'). Any other columns in the
+table will be ignored. You must furthermore create an index as follows:
+
+```
+CREATE INDEX ON secondary_importance USING gist(ST_ConvexHull(gist))
+```
+
+The following raster2pgsql command will create a table that conforms to
+the requirements:
+
+```
+raster2pgsql -I -C -Y -d -t 128x128 input.tiff public.secondary_importance
+```

docs/customize/Settings.md

@@ -148,6 +148,29 @@ Setting this option to 'yes' means that Nominatim skips reindexing of contained
 objects when the area becomes too large.
 
 
+#### NOMINATIM_UPDATE_FORWARD_DEPENDENCIES
+
+| Summary            |                                                     |
+| --------------     | --------------------------------------------------- |
+| **Description:**   | Forward geometry changes to dependet objects |
+| **Format:**        | bool |
+| **Default:**       | no |
+| **Comment:**       | EXPERT ONLY. Must not be enabled after import. |
+
+The geometry of OSM ways and relations may change when a node that is part
+of the object is moved around. These changes are not propagated per default.
+The geometry of ways/relations is only updated the next time that the object
+itself is touched. When enabling this option, then dependent objects will
+be marked for update when one of its member objects changes.
+
+Enabling this option may slow down updates significantly.
+
+!!! warning
+    If you want to enable this option, it must be set already on import.
+    Do not enable this option on an existing database that was imported with
+    NOMINATIM_UPDATE_FORWARD_DEPENDENCIES=no.
+    Updates will become unusably slow.
+
 #### NOMINATIM_LANGUAGES
 
 | Summary            |                                                     |

docs/customize/Tokenizers.md

@@ -213,6 +213,15 @@ The following is a list of sanitizers that are shipped with Nominatim.
     rendering:
         heading_level: 6
 
+##### clean-tiger-tags
+
+::: nominatim.tokenizer.sanitizers.clean_tiger_tags
+    selection:
+        members: False
+    rendering:
+        heading_level: 6
+
+
 
 #### Token Analysis
 

docs/mkdocs.yml

@@ -30,6 +30,7 @@ nav:
         - 'Configuration Settings': 'customize/Settings.md'
         - 'Per-Country Data': 'customize/Country-Settings.md'
         - 'Place Ranking' : 'customize/Ranking.md'
+        - 'Importance' : 'customize/Importance.md'
         - 'Tokenizers' : 'customize/Tokenizers.md'
         - 'Special Phrases': 'customize/Special-Phrases.md'
         - 'External data: US housenumbers from TIGER': 'customize/Tiger.md'

lib-php/PlaceLookup.php

@@ -187,12 +187,12 @@ class PlaceLookup
             return null;
         }
 
-        $aResults = $this->lookup(array($iPlaceID => new Result($iPlaceID)));
+        $aResults = $this->lookup(array($iPlaceID => new Result($iPlaceID)), 0, 30, true);
 
         return empty($aResults) ? null : reset($aResults);
     }
 
-    public function lookup($aResults, $iMinRank = 0, $iMaxRank = 30)
+    public function lookup($aResults, $iMinRank = 0, $iMaxRank = 30, $bAllowLinked = false)
     {
         Debug::newFunction('Place lookup');
 
@@ -247,7 +247,9 @@ class PlaceLookup
             if ($this->sAllowedTypesSQLList) {
                 $sSQL .= 'AND placex.class in '.$this->sAllowedTypesSQLList;
             }
-            $sSQL .= '    AND linked_place_id is null ';
+            if (!$bAllowLinked) {
+                $sSQL .= '    AND linked_place_id is null ';
+            }
             $sSQL .= ' GROUP BY ';
             $sSQL .= '     osm_type, ';
             $sSQL .= '     osm_id, ';

lib-php/ReverseGeocode.php

@@ -189,14 +189,16 @@ class ReverseGeocode
         $sSQL .= '(select place_id, parent_place_id, rank_address, rank_search, country_code, geometry';
         $sSQL .= ' FROM placex';
         $sSQL .= ' WHERE ST_GeometryType(geometry) in (\'ST_Polygon\', \'ST_MultiPolygon\')';
-        $sSQL .= ' AND rank_address Between 5 AND ' .$iMaxRank;
+        // Ensure that query planner doesn't use the index on rank_search.
+        $sSQL .= ' AND coalesce(rank_search, 0) between 5 and ' .$iMaxRank;
+        $sSQL .= ' AND rank_address between 4 and 25'; // needed for index selection
         $sSQL .= ' AND geometry && '.$sPointSQL;
         $sSQL .= ' AND type != \'postcode\' ';
         $sSQL .= ' AND name is not null';
         $sSQL .= ' AND indexed_status = 0 and linked_place_id is null';
-        $sSQL .= ' ORDER BY rank_address DESC LIMIT 50 ) as a';
+        $sSQL .= ' ORDER BY rank_search DESC LIMIT 50 ) as a';
-        $sSQL .= ' WHERE ST_CONTAINS(geometry, '.$sPointSQL.' )';
+        $sSQL .= ' WHERE ST_Contains(geometry, '.$sPointSQL.' )';
-        $sSQL .= ' ORDER BY rank_address DESC LIMIT 1';
+        $sSQL .= ' ORDER BY rank_search DESC LIMIT 1';
         Debug::printSQL($sSQL);
 
         $aPoly = $this->oDB->getRow($sSQL, null, 'Could not determine polygon containing the point.');
@@ -208,7 +210,7 @@ class ReverseGeocode
             $iRankSearch = $aPoly['rank_search'];
             $iPlaceID = $aPoly['place_id'];
 
-            if ($iRankAddress != $iMaxRank) {
+            if ($iRankSearch != $iMaxRank) {
                 $sSQL = 'SELECT place_id FROM ';
                 $sSQL .= '(SELECT place_id, rank_search, country_code, geometry,';
                 $sSQL .= ' ST_distance('.$sPointSQL.', geometry) as distance';

lib-sql/functions/importance.sql

@@ -100,32 +100,55 @@ LANGUAGE plpgsql STABLE;
 
 CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE,
                                               country_code varchar(2),
-                                              osm_type varchar(1), osm_id BIGINT)
+                                              rank_search SMALLINT,
+                                              centroid GEOMETRY)
   RETURNS place_importance
   AS $$
 DECLARE
   match RECORD;
   result place_importance;
+  osm_views_exists BIGINT;
+  views BIGINT;
 BEGIN
-  FOR match IN SELECT * FROM get_wikipedia_match(extratags, country_code)
+  -- add importance by wikipedia article if the place has one
-               WHERE language is not NULL
+  FOR match IN
+    SELECT * FROM get_wikipedia_match(extratags, country_code)
+    WHERE language is not NULL
   LOOP
     result.importance := match.importance;
     result.wikipedia := match.language || ':' || match.title;
     RETURN result;
   END LOOP;
 
-  IF extratags ? 'wikidata' THEN
+  -- Nothing? Then try with the wikidata tag.
+  IF result.importance is null AND extratags ? 'wikidata' THEN
     FOR match IN SELECT * FROM wikipedia_article
                   WHERE wd_page_title = extratags->'wikidata'
-                  ORDER BY language = 'en' DESC, langcount DESC LIMIT 1 LOOP
+                  ORDER BY language = 'en' DESC, langcount DESC LIMIT 1
+    LOOP
       result.importance := match.importance;
       result.wikipedia := match.language || ':' || match.title;
       RETURN result;
     END LOOP;
   END IF;
 
-  RETURN null;
+  -- Still nothing? Fall back to a default.
+  IF result.importance is null THEN
+    result.importance := 0.75001 - (rank_search::float / 40);
+  END IF;
+
+{% if 'secondary_importance' in db.tables %}
+  FOR match IN
+    SELECT ST_Value(rast, centroid) as importance
+    FROM secondary_importance
+    WHERE ST_Intersects(ST_ConvexHull(rast), centroid) LIMIT 1
+  LOOP
+    -- Secondary importance as tie breaker with 0.0001 weight.
+    result.importance := result.importance + match.importance::float / 655350000;
+  END LOOP;
+{% endif %}
+
+  RETURN result;
 END;
 $$
 LANGUAGE plpgsql;

lib-sql/functions/interpolation.sql

@@ -52,7 +52,9 @@ BEGIN
 
   IF parent_place_id is null THEN
     FOR location IN SELECT place_id FROM placex
-        WHERE ST_DWithin(geom, placex.geometry, 0.001) and placex.rank_search = 26
+        WHERE ST_DWithin(geom, placex.geometry, 0.001)
+              and placex.rank_search = 26
+              and placex.osm_type = 'W' -- needed for index selection
         ORDER BY CASE WHEN ST_GeometryType(geom) = 'ST_Line' THEN
                   (ST_distance(placex.geometry, ST_LineInterpolatePoint(geom,0))+
                   ST_distance(placex.geometry, ST_LineInterpolatePoint(geom,0.5))+

lib-sql/functions/place_triggers.sql

@@ -34,6 +34,11 @@ BEGIN
     RETURN null;
   END IF;
 
+  -- Remove the place from the list of places to be deleted
+  DELETE FROM place_to_be_deleted pdel
+    WHERE pdel.osm_type = NEW.osm_type and pdel.osm_id = NEW.osm_id
+          and pdel.class = NEW.class;
+
   -- Have we already done this place?
   SELECT * INTO existing
     FROM place
@@ -42,8 +47,6 @@ BEGIN
 
   {% if debug %}RAISE WARNING 'Existing: %',existing.osm_id;{% endif %}
 
-  -- Handle a place changing type by removing the old data.
-  -- (This trigger is executed BEFORE INSERT of the NEW tuple.)
   IF existing.osm_type IS NULL THEN
     DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class;
   END IF;
@@ -187,15 +190,11 @@ BEGIN
     END IF;
     {% endif %}
 
-    IF existing.osm_type IS NOT NULL THEN
+    IF existingplacex.osm_type is not NULL THEN
-      -- Pathological case caused by the triggerless copy into place during initial import
+      -- Mark any existing place for delete in the placex table
-      -- force delete even for large areas, it will be reinserted later
+      UPDATE placex SET indexed_status = 100
-      UPDATE place SET geometry = ST_SetSRID(ST_Point(0,0), 4326)
+        WHERE placex.osm_type = NEW.osm_type and placex.osm_id = NEW.osm_id
-        WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
+              and placex.class = NEW.class and placex.type = NEW.type;
-              and class = NEW.class and type = NEW.type;
-      DELETE FROM place
-        WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
-              and class = NEW.class and type = NEW.type;
     END IF;
 
     -- Process it as a new insertion
@@ -206,6 +205,27 @@ BEGIN
 
     {% if debug %}RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name;{% endif %}
 
+    IF existing.osm_type is not NULL THEN
+      -- If there is already an entry in place, just update that, if necessary.
+      IF coalesce(existing.name, ''::hstore) != coalesce(NEW.name, ''::hstore)
+         or coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore)
+         or coalesce(existing.extratags, ''::hstore) != coalesce(NEW.extratags, ''::hstore)
+         or coalesce(existing.admin_level, 15) != coalesce(NEW.admin_level, 15)
+         or existing.geometry::text != NEW.geometry::text
+      THEN
+        UPDATE place
+          SET name = NEW.name,
+              address = NEW.address,
+              extratags = NEW.extratags,
+              admin_level = NEW.admin_level,
+              geometry = NEW.geometry
+          WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
+                and class = NEW.class and type = NEW.type;
+      END IF;
+
+      RETURN NULL;
+    END IF;
+
     RETURN NEW;
   END IF;
 
@@ -321,35 +341,67 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
-
 CREATE OR REPLACE FUNCTION place_delete()
   RETURNS TRIGGER
   AS $$
 DECLARE
-  has_rank BOOLEAN;
+  deferred BOOLEAN;
 BEGIN
+  {% if debug %}RAISE WARNING 'Delete for % % %/%', OLD.osm_type, OLD.osm_id, OLD.class, OLD.type;{% endif %}
 
-  {% if debug %}RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type;{% endif %}
+  deferred := ST_IsValid(OLD.geometry) and ST_Area(OLD.geometry) > 2;
-
+  IF deferred THEN
-  -- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
+    SELECT bool_or(not (rank_address = 0 or rank_address > 25)) INTO deferred
-  IF st_area(OLD.geometry) > 2 and st_isvalid(OLD.geometry) THEN
+      FROM placex
-    SELECT bool_or(not (rank_address = 0 or rank_address > 25)) as ranked FROM placex WHERE osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type INTO has_rank;
+      WHERE osm_type = OLD.osm_type and osm_id = OLD.osm_id
-    IF has_rank THEN
+            and class = OLD.class and type = OLD.type;
-      insert into import_polygon_delete (osm_type, osm_id, class, type) values (OLD.osm_type,OLD.osm_id,OLD.class,OLD.type);
-      RETURN NULL;
-    END IF;
   END IF;
 
-  -- mark for delete
+  INSERT INTO place_to_be_deleted (osm_type, osm_id, class, type, deferred)
-  UPDATE placex set indexed_status = 100 where osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type;
+    VALUES(OLD.osm_type, OLD.osm_id, OLD.class, OLD.type, deferred);
 
-  -- interpolations are special
+  RETURN NULL;
-  IF OLD.osm_type='W' and OLD.class = 'place' and OLD.type = 'houses' THEN
-    UPDATE location_property_osmline set indexed_status = 100 where osm_id = OLD.osm_id; -- osm_id = wayid (=old.osm_id)
-  END IF;
-
-  RETURN OLD;
 END;
-$$
+$$ LANGUAGE plpgsql;
-LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION flush_deleted_places()
+  RETURNS INTEGER
+  AS $$
+BEGIN
+  -- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
+  INSERT INTO import_polygon_delete (osm_type, osm_id, class, type)
+    SELECT osm_type, osm_id, class, type FROM place_to_be_deleted WHERE deferred;
+
+  -- delete from place table
+  ALTER TABLE place DISABLE TRIGGER place_before_delete;
+  DELETE FROM place USING place_to_be_deleted
+    WHERE place.osm_type = place_to_be_deleted.osm_type
+          and place.osm_id = place_to_be_deleted.osm_id
+          and place.class = place_to_be_deleted.class
+          and place.type = place_to_be_deleted.type
+          and not deferred;
+  ALTER TABLE place ENABLE TRIGGER place_before_delete;
+
+  -- Mark for delete in the placex table
+  UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
+    WHERE placex.osm_type = place_to_be_deleted.osm_type
+          and placex.osm_id = place_to_be_deleted.osm_id
+          and placex.class = place_to_be_deleted.class
+          and placex.type = place_to_be_deleted.type
+          and not deferred;
+
+   -- Mark for delete in interpolations
+   UPDATE location_property_osmline SET indexed_status = 100 FROM place_to_be_deleted
+    WHERE place_to_be_deleted.osm_type = 'W'
+          and place_to_be_deleted.class = 'place'
+          and place_to_be_deleted.type = 'houses'
+          and location_property_osmline.osm_id = place_to_be_deleted.osm_id
+          and not deferred;
+
+   -- Clear todo list.
+   TRUNCATE TABLE place_to_be_deleted;
+
+   RETURN NULL;
+END;
+$$ LANGUAGE plpgsql;
 

lib-sql/functions/placex_triggers.sql

@@ -107,12 +107,17 @@ LANGUAGE plpgsql STABLE;
 
 
 CREATE OR REPLACE FUNCTION find_associated_street(poi_osm_type CHAR(1),
-                                                  poi_osm_id BIGINT)
+                                                  poi_osm_id BIGINT,
+                                                  bbox GEOMETRY)
   RETURNS BIGINT
   AS $$
 DECLARE
   location RECORD;
   parent RECORD;
+  result BIGINT;
+  distance FLOAT;
+  new_distance FLOAT;
+  waygeom GEOMETRY;
 BEGIN
   FOR location IN
     SELECT members FROM planet_osm_rels
@@ -123,19 +128,34 @@ BEGIN
     FOR i IN 1..array_upper(location.members, 1) BY 2 LOOP
       IF location.members[i+1] = 'street' THEN
         FOR parent IN
-          SELECT place_id from placex
+          SELECT place_id, geometry
+           FROM placex
            WHERE osm_type = upper(substring(location.members[i], 1, 1))::char(1)
                  and osm_id = substring(location.members[i], 2)::bigint
                  and name is not null
                  and rank_search between 26 and 27
         LOOP
-          RETURN parent.place_id;
+          -- Find the closest 'street' member.
+          -- Avoid distance computation for the frequent case where there is
+          -- only one street member.
+          IF waygeom is null THEN
+            result := parent.place_id;
+            waygeom := parent.geometry;
+          ELSE
+            distance := coalesce(distance, ST_Distance(waygeom, bbox));
+            new_distance := ST_Distance(parent.geometry, bbox);
+            IF new_distance < distance THEN
+              distance := new_distance;
+              result := parent.place_id;
+              waygeom := parent.geometry;
+            END IF;
+          END IF;
         END LOOP;
       END IF;
     END LOOP;
   END LOOP;
 
-  RETURN NULL;
+  RETURN result;
 END;
 $$
 LANGUAGE plpgsql STABLE;
@@ -162,7 +182,7 @@ BEGIN
   {% if debug %}RAISE WARNING 'finding street for % %', poi_osm_type, poi_osm_id;{% endif %}
 
   -- Is this object part of an associatedStreet relation?
-  parent_place_id := find_associated_street(poi_osm_type, poi_osm_id);
+  parent_place_id := find_associated_street(poi_osm_type, poi_osm_id, bbox);
 
   IF parent_place_id is null THEN
     parent_place_id := find_parent_for_address(token_info, poi_partition, bbox);
@@ -185,7 +205,7 @@ BEGIN
         RETURN location.place_id;
       END IF;
 
-      parent_place_id := find_associated_street('W', location.osm_id);
+      parent_place_id := find_associated_street('W', location.osm_id, bbox);
     END LOOP;
   END IF;
 
@@ -197,6 +217,7 @@ BEGIN
         SELECT place_id FROM placex
          WHERE bbox && geometry AND _ST_Covers(geometry, ST_Centroid(bbox))
                AND rank_address between 5 and 25
+               AND ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon')
          ORDER BY rank_address desc
       LOOP
         RETURN location.place_id;
@@ -212,6 +233,7 @@ BEGIN
         SELECT place_id FROM placex
          WHERE bbox && geometry AND _ST_Covers(geometry, ST_Centroid(bbox))
                AND rank_address between 5 and 25
+               AND ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon')
         ORDER BY rank_address desc
       LOOP
         RETURN location.place_id;
@@ -275,7 +297,9 @@ BEGIN
 
   -- If extratags has a place tag, look for linked nodes by their place type.
   -- Area and node still have to have the same name.
-  IF bnd.extratags ? 'place' and bnd_name is not null THEN
+  IF bnd.extratags ? 'place' and bnd.extratags->'place' != 'postcode'
+     and bnd_name is not null
+  THEN
     FOR linked_placex IN
       SELECT * FROM placex
       WHERE (position(lower(name->'name') in bnd_name) > 0
@@ -284,7 +308,6 @@ BEGIN
         AND placex.osm_type = 'N'
         AND (placex.linked_place_id is null or placex.linked_place_id = bnd.place_id)
         AND placex.rank_search < 26 -- needed to select the right index
-        AND placex.type != 'postcode'
         AND ST_Covers(bnd.geometry, placex.geometry)
     LOOP
       {% if debug %}RAISE WARNING 'Found type-matching place node %', linked_placex.osm_id;{% endif %}
@@ -846,7 +869,8 @@ BEGIN
       FROM placex
       WHERE osm_type = 'R' and class = 'boundary' and type = 'administrative'
             and admin_level < NEW.admin_level and admin_level > 3
-            and rank_address > 0
+            and rank_address between 1 and 25 -- for index selection
+            and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') -- for index selection
             and geometry && NEW.centroid and _ST_Covers(geometry, NEW.centroid)
       ORDER BY admin_level desc LIMIT 1
     LOOP
@@ -874,8 +898,9 @@ BEGIN
           FROM placex,
                LATERAL compute_place_rank(country_code, 'A', class, type,
                                           admin_level, False, null) prank
-          WHERE class = 'place' and rank_address < 24
+          WHERE class = 'place' and rank_address between 1 and 23
                 and prank.address_rank >= NEW.rank_address
+                and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') -- select right index
                 and geometry && NEW.geometry
                 and geometry ~ NEW.geometry -- needed because ST_Relate does not do bbox cover test
                 and ST_Relate(geometry, NEW.geometry, 'T*T***FF*') -- contains but not equal
@@ -896,6 +921,8 @@ BEGIN
                LATERAL compute_place_rank(country_code, 'A', class, type,
                                           admin_level, False, null) prank
           WHERE prank.address_rank < 24
+                and rank_address between 1 and 25 -- select right index
+                and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') -- select right index
                 and prank.address_rank >= NEW.rank_address
                 and geometry && NEW.geometry
                 and geometry ~ NEW.geometry -- needed because ST_Relate does not do bbox cover test
@@ -916,6 +943,8 @@ BEGIN
              LATERAL compute_place_rank(country_code, 'A', class, type,
                                         admin_level, False, null) prank
         WHERE osm_type = 'R'
+              and rank_address between 1 and 25 -- select right index
+              and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') -- select right index
               and ((class = 'place' and prank.address_rank = NEW.rank_address)
                    or (class = 'boundary' and rank_address = NEW.rank_address))
               and geometry && NEW.centroid and _ST_Covers(geometry, NEW.centroid)
@@ -956,7 +985,7 @@ BEGIN
 
   NEW.importance := null;
   SELECT wikipedia, importance
-    FROM compute_importance(NEW.extratags, NEW.country_code, NEW.osm_type, NEW.osm_id)
+    FROM compute_importance(NEW.extratags, NEW.country_code, NEW.rank_search, NEW.centroid)
     INTO NEW.wikipedia,NEW.importance;
 
 {% if debug %}RAISE WARNING 'Importance computed from wikipedia: %', NEW.importance;{% endif %}
@@ -1038,7 +1067,7 @@ BEGIN
   IF linked_place is not null THEN
     -- Recompute the ranks here as the ones from the linked place might
     -- have been shifted to accommodate surrounding boundaries.
-    SELECT place_id, osm_id, class, type, extratags,
+    SELECT place_id, osm_id, class, type, extratags, rank_search,
            centroid, geometry,
            (compute_place_rank(country_code, osm_type, class, type, admin_level,
                               (extratags->'capital') = 'yes', null)).*
@@ -1079,7 +1108,7 @@ BEGIN
 
     SELECT wikipedia, importance
       FROM compute_importance(location.extratags, NEW.country_code,
-                              'N', location.osm_id)
+                              location.rank_search, NEW.centroid)
       INTO linked_wikipedia,linked_importance;
 
     -- Use the maximum importance if one could be computed from the linked object.

lib-sql/indices.sql

@@ -10,65 +10,86 @@
 
 CREATE INDEX IF NOT EXISTS idx_place_addressline_address_place_id
   ON place_addressline USING BTREE (address_place_id) {{db.tablespace.search_index}};
-
+---
 CREATE INDEX IF NOT EXISTS idx_placex_rank_search
   ON placex USING BTREE (rank_search) {{db.tablespace.search_index}};
-
+---
 CREATE INDEX IF NOT EXISTS idx_placex_rank_address
   ON placex USING BTREE (rank_address) {{db.tablespace.search_index}};
-
+---
 CREATE INDEX IF NOT EXISTS idx_placex_parent_place_id
   ON placex USING BTREE (parent_place_id) {{db.tablespace.search_index}}
   WHERE parent_place_id IS NOT NULL;
-
+---
+CREATE INDEX IF NOT EXISTS idx_placex_geometry ON placex
+  USING GIST (geometry) {{db.tablespace.search_index}};
+---
 CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPolygon
   ON placex USING gist (geometry) {{db.tablespace.search_index}}
   WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')
     AND rank_address between 4 and 25 AND type != 'postcode'
     AND name is not null AND indexed_status = 0 AND linked_place_id is null;
-
+---
 CREATE INDEX IF NOT EXISTS idx_osmline_parent_place_id
   ON location_property_osmline USING BTREE (parent_place_id) {{db.tablespace.search_index}}
   WHERE parent_place_id is not null;
-
+---
 CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id
   ON location_property_osmline USING BTREE (osm_id) {{db.tablespace.search_index}};
-
+---
 CREATE INDEX IF NOT EXISTS idx_postcode_postcode
   ON location_postcode USING BTREE (postcode) {{db.tablespace.search_index}};
 
+{% if drop %}
+---
+  DROP INDEX IF EXISTS idx_placex_geometry_address_area_candidates;
+  DROP INDEX IF EXISTS idx_placex_geometry_buildings;
+  DROP INDEX IF EXISTS idx_placex_geometry_lower_rank_ways;
+  DROP INDEX IF EXISTS idx_placex_wikidata;
+  DROP INDEX IF EXISTS idx_placex_rank_address_sector;
+  DROP INDEX IF EXISTS idx_placex_rank_boundaries_sector;
+{% else %}
 -- Indices only needed for updating.
-
+---
-{% if not drop %}
-  CREATE INDEX IF NOT EXISTS idx_placex_pendingsector
-    ON placex USING BTREE (rank_address,geometry_sector) {{db.tablespace.address_index}}
-    WHERE indexed_status > 0;
-
   CREATE INDEX IF NOT EXISTS idx_location_area_country_place_id
     ON location_area_country USING BTREE (place_id) {{db.tablespace.address_index}};
-
+---
   CREATE UNIQUE INDEX IF NOT EXISTS idx_place_osm_unique
     ON place USING btree(osm_id, osm_type, class, type) {{db.tablespace.address_index}};
+---
+-- Table needed for running updates with osm2pgsql on place.
+  CREATE TABLE IF NOT EXISTS place_to_be_deleted (
+    osm_type CHAR(1),
+    osm_id BIGINT,
+    class TEXT,
+    type TEXT,
+    deferred BOOLEAN
+   );
 {% endif %}
 
 -- Indices only needed for search.
-
 {% if 'search_name' in db.tables %}
+---
   CREATE INDEX IF NOT EXISTS idx_search_name_nameaddress_vector
     ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
+---
   CREATE INDEX IF NOT EXISTS idx_search_name_name_vector
     ON search_name USING GIN (name_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
+---
   CREATE INDEX IF NOT EXISTS idx_search_name_centroid
     ON search_name USING GIST (centroid) {{db.tablespace.search_index}};
 
   {% if postgres.has_index_non_key_column %}
+---
     CREATE INDEX IF NOT EXISTS idx_placex_housenumber
       ON placex USING btree (parent_place_id)
       INCLUDE (housenumber) {{db.tablespace.search_index}}
       WHERE housenumber is not null;
+---
     CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id_with_hnr
       ON location_property_osmline USING btree(parent_place_id)
       INCLUDE (startnumber, endnumber) {{db.tablespace.search_index}}
       WHERE startnumber is not null;
   {% endif %}
+
 {% endif %}

lib-sql/tables.sql

@@ -137,7 +137,9 @@ CREATE TABLE place_addressline (
   ) {{db.tablespace.search_data}};
 CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {{db.tablespace.search_index}};
 
-drop table if exists placex;
+---------  PLACEX - storage for all indexed places -----------------
+
+DROP TABLE IF EXISTS placex;
 CREATE TABLE placex (
   place_id BIGINT NOT NULL,
   parent_place_id BIGINT,
@@ -157,20 +159,66 @@ CREATE TABLE placex (
   postcode TEXT,
   centroid GEOMETRY(Geometry, 4326)
   ) {{db.tablespace.search_data}};
+
 CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {{db.tablespace.search_index}};
-CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {{db.tablespace.search_index}};
+{% for osm_type in ('N', 'W', 'R') %}
-CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {{db.tablespace.address_index}} WHERE linked_place_id IS NOT NULL;
+CREATE INDEX idx_placex_osmid_{{osm_type | lower}} ON placex
-CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}};
+  USING BTREE (osm_id) {{db.tablespace.search_index}}
-CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {{db.tablespace.search_index}};
+  WHERE osm_type = '{{osm_type}}';
+{% endfor %}
+
+-- Usage: - removing linkage status on update
+--        - lookup linked places for /details
+CREATE INDEX idx_placex_linked_place_id ON placex
+  USING BTREE (linked_place_id) {{db.tablespace.address_index}}
+  WHERE linked_place_id IS NOT NULL;
+
+-- Usage: - check that admin boundaries do not overtake each other rank-wise
+--        - check that place node in a admin boundary with the same address level
+--        - boundary is not completely contained in a place area
+--        - parenting of large-area or unparentable features
+CREATE INDEX idx_placex_geometry_address_area_candidates ON placex
+  USING gist (geometry) {{db.tablespace.address_index}}
+  WHERE rank_address between 1 and 25
+        and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon');
+
+-- Usage: - POI is within building with housenumber
 CREATE INDEX idx_placex_geometry_buildings ON placex
-  USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.search_index}}
+  USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.address_index}}
   WHERE address is not null and rank_search = 30
         and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon');
+
+-- Usage: - linking of similar named places to boundaries
+--        - linking of place nodes with same type to boundaries
+--        - lookupPolygon()
 CREATE INDEX idx_placex_geometry_placenode ON placex
-  USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.search_index}}
+  USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.address_index}}
   WHERE osm_type = 'N' and rank_search < 26
-        and class = 'place' and type != 'postcode' and linked_place_id is null;
+        and class = 'place' and type != 'postcode';
-CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26;
+
+-- Usage: - is node part of a way?
+--        - find parent of interpolation spatially
+CREATE INDEX idx_placex_geometry_lower_rank_ways ON placex
+  USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.address_index}}
+  WHERE osm_type = 'W' and rank_search >= 26;
+
+-- Usage: - linking place nodes by wikidata tag to boundaries
+CREATE INDEX idx_placex_wikidata on placex
+  USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}}
+  WHERE extratags ? 'wikidata' and class = 'place'
+        and osm_type = 'N' and rank_search < 26;
+
+-- The following two indexes function as a todo list for indexing.
+
+CREATE INDEX idx_placex_rank_address_sector ON placex
+  USING BTREE (rank_address, geometry_sector) {{db.tablespace.address_index}}
+  WHERE indexed_status > 0;
+
+CREATE INDEX idx_placex_rank_boundaries_sector ON placex
+  USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}}
+  WHERE class = 'boundary' and type = 'administrative'
+        and indexed_status > 0;
+
 
 DROP SEQUENCE IF EXISTS seq_place;
 CREATE SEQUENCE seq_place start 1;
@@ -228,7 +276,7 @@ CREATE SEQUENCE file start 1;
 
 -- null table so it won't error
 -- deliberately no drop - importing the table is expensive and static, if it is already there better to avoid removing it
-CREATE TABLE wikipedia_article (
+CREATE TABLE IF NOT EXISTS wikipedia_article (
     language text NOT NULL,
     title text NOT NULL,
     langcount integer,
@@ -242,15 +290,12 @@ CREATE TABLE wikipedia_article (
     wd_page_title text,
     instance_of text
 );
-ALTER TABLE ONLY wikipedia_article ADD CONSTRAINT wikipedia_article_pkey PRIMARY KEY (language, title);
-CREATE INDEX idx_wikipedia_article_osm_id ON wikipedia_article USING btree (osm_type, osm_id);
 
-CREATE TABLE wikipedia_redirect (
+CREATE TABLE IF NOT EXISTS wikipedia_redirect (
     language text,
     from_title text,
     to_title text
 );
-ALTER TABLE ONLY wikipedia_redirect ADD CONSTRAINT wikipedia_redirect_pkey PRIMARY KEY (language, from_title);
 
 -- osm2pgsql does not create indexes on the middle tables for Nominatim
 -- Add one for lookup of associated street relations.

module/CMakeLists.txt

@@ -1,6 +1,6 @@
 # just use the pgxs makefile
 
-foreach(suffix ${PostgreSQL_ADDITIONAL_VERSIONS} "14" "13" "12" "11" "10" "9.6")
+foreach(suffix ${PostgreSQL_ADDITIONAL_VERSIONS} "15" "14" "13" "12" "11" "10" "9.6")
     list(APPEND PG_CONFIG_HINTS
          "/usr/pgsql-${suffix}/bin")
 endforeach()

nominatim/clicmd/admin.py

@@ -20,6 +20,7 @@ from nominatim.clicmd.args import NominatimArgs
 
 LOG = logging.getLogger()
 
+
 class AdminFuncs:
     """\
     Analyse and maintain the database.
@@ -36,6 +37,8 @@ class AdminFuncs:
                           help='Migrate the database to a new software version')
         objs.add_argument('--analyse-indexing', action='store_true',
                           help='Print performance analysis of the indexing process')
+        objs.add_argument('--collect-os-info', action="store_true",
+                          help="Generate a report about the host system information")
         group = parser.add_argument_group('Arguments for cache warming')
         group.add_argument('--search-only', action='store_const', dest='target',
                            const='search',
@@ -70,8 +73,13 @@ class AdminFuncs:
             from ..tools import migration
             return migration.migrate(args.config, args)
 
-        return 1
+        if args.collect_os_info:
+            LOG.warning("Reporting System Information")
+            from ..tools import collect_os_info
+            collect_os_info.report_system_information(args.config)
+            return 0
 
+        return 1
 
     def _warm(self, args: NominatimArgs) -> int:
         LOG.warning('Warming database caches')

nominatim/clicmd/args.py

@@ -76,6 +76,7 @@ class NominatimArgs:
     warm: bool
     check_database: bool
     migrate: bool
+    collect_os_info: bool
     analyse_indexing: bool
     target: Optional[str]
     osm_id: Optional[str]
@@ -114,6 +115,7 @@ class NominatimArgs:
     address_levels: bool
     functions: bool
     wiki_data: bool
+    secondary_importance: bool
     importance: bool
     website: bool
     diffs: bool
@@ -182,8 +184,10 @@ class NominatimArgs:
         return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.osm2pgsql_path,
                     osm2pgsql_cache=self.osm2pgsql_cache or default_cache,
                     osm2pgsql_style=self.config.get_import_style_file(),
+                    osm2pgsql_style_path=self.config.config_dir,
                     threads=self.threads or default_threads,
                     dsn=self.config.get_libpq_dsn(),
+                    forward_dependencies=self.config.get_bool('UPDATE_FORWARD_DEPENDENCIES'),
                     flatnode_file=str(self.config.get_path('FLATNODE_FILE') or ''),
                     tablespaces=dict(slim_data=self.config.TABLESPACE_OSM_DATA,
                                      slim_index=self.config.TABLESPACE_OSM_INDEX,

nominatim/clicmd/refresh.py

@@ -63,6 +63,8 @@ class UpdateRefresh:
                            help='Update the PL/pgSQL functions in the database')
         group.add_argument('--wiki-data', action='store_true',
                            help='Update Wikipedia/data importance numbers')
+        group.add_argument('--secondary-importance', action='store_true',
+                           help='Update secondary importance raster data')
         group.add_argument('--importance', action='store_true',
                            help='Recompute place importances (expensive!)')
         group.add_argument('--website', action='store_true',
@@ -83,7 +85,7 @@ class UpdateRefresh:
                            help='Enable debug warning statements in functions')
 
 
-    def run(self, args: NominatimArgs) -> int: #pylint: disable=too-many-branches
+    def run(self, args: NominatimArgs) -> int: #pylint: disable=too-many-branches, too-many-statements
         from ..tools import refresh, postcodes
         from ..indexer.indexer import Indexer
 
@@ -115,6 +117,20 @@ class UpdateRefresh:
             with connect(args.config.get_libpq_dsn()) as conn:
                 refresh.load_address_levels_from_config(conn, args.config)
 
+        # Attention: must come BEFORE functions
+        if args.secondary_importance:
+            with connect(args.config.get_libpq_dsn()) as conn:
+                # If the table did not exist before, then the importance code
+                # needs to be enabled.
+                if not conn.table_exists('secondary_importance'):
+                    args.functions = True
+
+            LOG.warning('Import secondary importance raster data from %s', args.project_dir)
+            if refresh.import_secondary_importance(args.config.get_libpq_dsn(),
+                                                args.project_dir) > 0:
+                LOG.fatal('FATAL: Cannot update sendary importance raster data')
+                return 1
+
         if args.functions:
             LOG.warning('Create functions')
             with connect(args.config.get_libpq_dsn()) as conn:

nominatim/clicmd/setup.py

@@ -15,7 +15,7 @@ from pathlib import Path
 import psutil
 
 from nominatim.config import Configuration
-from nominatim.db.connection import connect, Connection
+from nominatim.db.connection import connect
 from nominatim.db import status, properties
 from nominatim.tokenizer.base import AbstractTokenizer
 from nominatim.version import version_str
@@ -59,7 +59,7 @@ class SetupAll:
                            help="Do not keep tables that are only needed for "
                                 "updating the database later")
         group2.add_argument('--offline', action='store_true',
-                           help="Do not attempt to load any additional data from the internet")
+                            help="Do not attempt to load any additional data from the internet")
         group3 = parser.add_argument_group('Expert options')
         group3.add_argument('--ignore-errors', action='store_true',
                            help='Continue import even when errors in SQL are present')
@@ -72,6 +72,8 @@ class SetupAll:
         from ..tools import database_import, refresh, postcodes, freeze
         from ..indexer.indexer import Indexer
 
+        num_threads = args.threads or psutil.cpu_count() or 1
+
         country_info.setup_country_config(args.config)
 
         if args.continue_at is None:
@@ -94,14 +96,21 @@ class SetupAll:
                                             drop=args.no_updates,
                                             ignore_errors=args.ignore_errors)
 
-            self._setup_tables(args.config, args.reverse_only)
-
             LOG.warning('Importing wikipedia importance data')
             data_path = Path(args.config.WIKIPEDIA_DATA_PATH or args.project_dir)
             if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),
                                                  data_path) > 0:
                 LOG.error('Wikipedia importance dump file not found. '
-                          'Will be using default importances.')
+                          'Calculating importance values of locations will not '
+                          'use Wikipedia importance data.')
+
+            LOG.warning('Importing secondary importance raster data')
+            if refresh.import_secondary_importance(args.config.get_libpq_dsn(),
+                                                   args.project_dir) != 0:
+                LOG.error('Secondary importance file not imported. '
+                          'Falling back to default ranking.')
+
+            self._setup_tables(args.config, args.reverse_only)
 
         if args.continue_at is None or args.continue_at == 'load-data':
             LOG.warning('Initialise tables')
@@ -109,8 +118,7 @@ class SetupAll:
                 database_import.truncate_data_tables(conn)
 
             LOG.warning('Load data into placex table')
-            database_import.load_data(args.config.get_libpq_dsn(),
+            database_import.load_data(args.config.get_libpq_dsn(), num_threads)
-                                      args.threads or psutil.cpu_count() or 1)
 
         LOG.warning("Setting up tokenizer")
         tokenizer = self._get_tokenizer(args.continue_at, args.config)
@@ -121,18 +129,15 @@ class SetupAll:
                                        args.project_dir, tokenizer)
 
         if args.continue_at is None or args.continue_at in ('load-data', 'indexing'):
-            if args.continue_at is not None and args.continue_at != 'load-data':
-                with connect(args.config.get_libpq_dsn()) as conn:
-                    self._create_pending_index(conn, args.config.TABLESPACE_ADDRESS_INDEX)
             LOG.warning('Indexing places')
-            indexer = Indexer(args.config.get_libpq_dsn(), tokenizer,
+            indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, num_threads)
-                              args.threads or psutil.cpu_count() or 1)
             indexer.index_full(analyse=not args.index_noanalyse)
 
         LOG.warning('Post-process tables')
         with connect(args.config.get_libpq_dsn()) as conn:
             database_import.create_search_indices(conn, args.config,
-                                                  drop=args.no_updates)
+                                                  drop=args.no_updates,
+                                                  threads=num_threads)
             LOG.warning('Create search index for default country names.')
             country_info.create_country_names(conn, tokenizer,
                                               args.config.get_str_list('LANGUAGES'))
@@ -188,27 +193,6 @@ class SetupAll:
         return tokenizer_factory.get_tokenizer_for_db(config)
 
 
-    def _create_pending_index(self, conn: Connection, tablespace: str) -> None:
-        """ Add a supporting index for finding places still to be indexed.
-
-            This index is normally created at the end of the import process
-            for later updates. When indexing was partially done, then this
-            index can greatly improve speed going through already indexed data.
-        """
-        if conn.index_exists('idx_placex_pendingsector'):
-            return
-
-        with conn.cursor() as cur:
-            LOG.warning('Creating support index')
-            if tablespace:
-                tablespace = 'TABLESPACE ' + tablespace
-            cur.execute(f"""CREATE INDEX idx_placex_pendingsector
-                            ON placex USING BTREE (rank_address,geometry_sector)
-                            {tablespace} WHERE indexed_status > 0
-                         """)
-        conn.commit()
-
-
     def _finalize_database(self, dsn: str, offline: bool) -> None:
         """ Determine the database date and set the status accordingly.
         """

nominatim/db/sql_preprocessor.py

@@ -11,6 +11,7 @@ from typing import Set, Dict, Any
 import jinja2
 
 from nominatim.db.connection import Connection
+from nominatim.db.async_connection import WorkerPool
 from nominatim.config import Configuration
 
 def _get_partitions(conn: Connection) -> Set[int]:
@@ -96,3 +97,21 @@ class SQLPreprocessor:
         with conn.cursor() as cur:
             cur.execute(sql)
         conn.commit()
+
+
+    def run_parallel_sql_file(self, dsn: str, name: str, num_threads: int = 1,
+                              **kwargs: Any) -> None:
+        """ Execure the given SQL files using parallel asynchronous connections.
+            The keyword arguments may supply additional parameters for
+            preprocessing.
+
+            After preprocessing the SQL code is cut at lines containing only
+            '---'. Each chunk is sent to one of the `num_threads` workers.
+        """
+        sql = self.env.get_template(name).render(**kwargs)
+
+        parts = sql.split('\n---\n')
+
+        with WorkerPool(dsn, num_threads) as pool:
+            for part in parts:
+                pool.next_free_worker().perform(part)

nominatim/indexer/indexer.py

@@ -128,58 +128,64 @@ class Indexer:
                     with conn.cursor() as cur:
                         cur.execute('ANALYZE')
 
-            self.index_by_rank(0, 4)
+            if self.index_by_rank(0, 4) > 0:
-            _analyze()
+                _analyze()
 
-            self.index_boundaries(0, 30)
+            if self.index_boundaries(0, 30) > 100:
-            _analyze()
+                _analyze()
 
-            self.index_by_rank(5, 25)
+            if self.index_by_rank(5, 25) > 100:
-            _analyze()
+                _analyze()
 
-            self.index_by_rank(26, 30)
+            if self.index_by_rank(26, 30) > 1000:
-            _analyze()
+                _analyze()
 
-            self.index_postcodes()
+            if self.index_postcodes() > 100:
-            _analyze()
+                _analyze()
 
 
-    def index_boundaries(self, minrank: int, maxrank: int) -> None:
+    def index_boundaries(self, minrank: int, maxrank: int) -> int:
         """ Index only administrative boundaries within the given rank range.
         """
+        total = 0
         LOG.warning("Starting indexing boundaries using %s threads",
                     self.num_threads)
 
         with self.tokenizer.name_analyzer() as analyzer:
             for rank in range(max(minrank, 4), min(maxrank, 26)):
-                self._index(runners.BoundaryRunner(rank, analyzer))
+                total += self._index(runners.BoundaryRunner(rank, analyzer))
 
-    def index_by_rank(self, minrank: int, maxrank: int) -> None:
+        return total
+
+    def index_by_rank(self, minrank: int, maxrank: int) -> int:
         """ Index all entries of placex in the given rank range (inclusive)
             in order of their address rank.
 
             When rank 30 is requested then also interpolations and
             places with address rank 0 will be indexed.
         """
+        total = 0
         maxrank = min(maxrank, 30)
         LOG.warning("Starting indexing rank (%i to %i) using %i threads",
                     minrank, maxrank, self.num_threads)
 
         with self.tokenizer.name_analyzer() as analyzer:
             for rank in range(max(1, minrank), maxrank + 1):
-                self._index(runners.RankRunner(rank, analyzer), 20 if rank == 30 else 1)
+                total += self._index(runners.RankRunner(rank, analyzer), 20 if rank == 30 else 1)
 
             if maxrank == 30:
-                self._index(runners.RankRunner(0, analyzer))
+                total += self._index(runners.RankRunner(0, analyzer))
-                self._index(runners.InterpolationRunner(analyzer), 20)
+                total += self._index(runners.InterpolationRunner(analyzer), 20)
+
+        return total
 
 
-    def index_postcodes(self) -> None:
+    def index_postcodes(self) -> int:
         """Index the entries of the location_postcode table.
         """
         LOG.warning("Starting indexing postcodes using %s threads", self.num_threads)
 
-        self._index(runners.PostcodeRunner(), 20)
+        return self._index(runners.PostcodeRunner(), 20)
 
 
     def update_status_table(self) -> None:
@@ -191,7 +197,7 @@ class Indexer:
 
             conn.commit()
 
-    def _index(self, runner: runners.Runner, batch: int = 1) -> None:
+    def _index(self, runner: runners.Runner, batch: int = 1) -> int:
         """ Index a single rank or table. `runner` describes the SQL to use
             for indexing. `batch` describes the number of objects that
             should be processed with a single SQL statement
@@ -233,4 +239,4 @@ class Indexer:
 
                 conn.commit()
 
-        progress.done()
+        return progress.done()

nominatim/indexer/progress.py

@@ -55,7 +55,7 @@ class ProgressLogger:
 
         self.next_info += int(places_per_sec) * self.log_interval
 
-    def done(self) -> None:
+    def done(self) -> int:
         """ Print final statistics about the progress.
         """
         rank_end_time = datetime.now()
@@ -70,3 +70,5 @@ class ProgressLogger:
         LOG.warning("Done %d/%d in %d @ %.3f per second - FINISHED %s\n",
                     self.done_places, self.total_places, int(diff_seconds),
                     places_per_sec, self.name)
+
+        return self.done_places

nominatim/tokenizer/sanitizers/clean_tiger_tags.py

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Sanitizer that preprocesses tags from the TIGER import.
+
+It makes the following changes:
+
+* remove state reference from tiger:county
+"""
+from typing import Callable
+import re
+
+from nominatim.tokenizer.sanitizers.base import ProcessInfo
+from nominatim.tokenizer.sanitizers.config import SanitizerConfig
+
+COUNTY_MATCH = re.compile('(.*), [A-Z][A-Z]')
+
+def _clean_tiger_county(obj: ProcessInfo) -> None:
+    """ Remove the state reference from tiger:county tags.
+
+        This transforms a name like 'Hamilton, AL' into 'Hamilton'.
+        If no state reference is detected at the end, the name is left as is.
+    """
+    if not obj.address:
+        return
+
+    for item in obj.address:
+        if item.kind == 'tiger' and item.suffix == 'county':
+            m = COUNTY_MATCH.fullmatch(item.name)
+            if m:
+                item.name = m[1]
+            # Switch kind and suffix, the split left them reversed.
+            item.kind = 'county'
+            item.suffix = 'tiger'
+
+            return
+
+
+def create(_: SanitizerConfig) -> Callable[[ProcessInfo], None]:
+    """ Create a housenumber processing function.
+    """
+    return _clean_tiger_county

nominatim/tools/check_database.py

@@ -114,9 +114,10 @@ def _get_indexes(conn: Connection) -> List[str]:
             indexes.extend(('idx_placex_housenumber',
                             'idx_osmline_parent_osm_id_with_hnr'))
     if conn.table_exists('place'):
-        indexes.extend(('idx_placex_pendingsector',
+        indexes.extend(('idx_location_area_country_place_id',
-                        'idx_location_area_country_place_id',
+                        'idx_place_osm_unique',
-                        'idx_place_osm_unique'))
+                        'idx_placex_rank_address_sector',
+                        'idx_placex_rank_boundaries_sector'))
 
     return indexes
 
@@ -199,7 +200,7 @@ def check_tokenizer(_: Connection, config: Configuration) -> CheckResult:
 def check_existance_wikipedia(conn: Connection, _: Configuration) -> CheckResult:
     """ Checking for wikipedia/wikidata data
     """
-    if not conn.table_exists('search_name'):
+    if not conn.table_exists('search_name') or not conn.table_exists('place'):
         return CheckState.NOT_APPLICABLE
 
     with conn.cursor() as cur:

nominatim/tools/collect_os_info.py

@@ -0,0 +1,167 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Collection of host system information including software versions, memory,
+storage, and database configuration.
+"""
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import List, Optional, Tuple, Union, cast
+
+import psutil
+from psycopg2.extensions import make_dsn, parse_dsn
+
+from nominatim.config import Configuration
+from nominatim.db.connection import connect
+from nominatim.typing import DictCursorResults
+from nominatim.version import version_str
+
+
+def convert_version(ver_tup: Tuple[int, int]) -> str:
+    """converts tuple version (ver_tup) to a string representation"""
+    return ".".join(map(str, ver_tup))
+
+
+def friendly_memory_string(mem: float) -> str:
+    """Create a user friendly string for the amount of memory specified as mem"""
+    mem_magnitude = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+    mag = 0
+    # determine order of magnitude
+    while mem > 1000:
+        mem /= 1000
+        mag += 1
+
+    return f"{mem:.1f} {mem_magnitude[mag]}"
+
+
+def run_command(cmd: Union[str, List[str]]) -> str:
+    """Runs a command using the shell and returns the output from stdout"""
+    try:
+        if sys.version_info < (3, 7):
+            cap_out = subprocess.run(cmd, stdout=subprocess.PIPE, check=False)
+        else:
+            cap_out = subprocess.run(cmd, capture_output=True, check=False)
+        return cap_out.stdout.decode("utf-8")
+    except FileNotFoundError:
+        # non-Linux system should end up here
+        return f"Unknown (unable to find the '{cmd}' command)"
+
+
+def os_name_info() -> str:
+    """Obtain Operating System Name (and possibly the version)"""
+    os_info = None
+    # man page os-release(5) details meaning of the fields
+    if Path("/etc/os-release").is_file():
+        os_info = from_file_find_line_portion(
+            "/etc/os-release", "PRETTY_NAME", "=")
+    # alternative location
+    elif Path("/usr/lib/os-release").is_file():
+        os_info = from_file_find_line_portion(
+            "/usr/lib/os-release", "PRETTY_NAME", "="
+        )
+
+    # fallback on Python's os name
+    if os_info is None or os_info == "":
+        os_info = os.name
+
+    # if the above is insufficient, take a look at neofetch's approach to OS detection
+    return os_info
+
+
+# Note: Intended to be used on informational files like /proc
+def from_file_find_line_portion(
+    filename: str, start: str, sep: str, fieldnum: int = 1
+) -> Optional[str]:
+    """open filename, finds the line starting with the 'start' string.
+    Splits the line using seperator and returns a "fieldnum" from the split."""
+    with open(filename, encoding='utf8') as file:
+        result = ""
+        for line in file:
+            if line.startswith(start):
+                result = line.split(sep)[fieldnum].strip()
+        return result
+
+
+def get_postgresql_config(version: int) -> str:
+    """Retrieve postgres configuration file"""
+    try:
+        with open(f"/etc/postgresql/{version}/main/postgresql.conf", encoding='utf8') as file:
+            db_config = file.read()
+            file.close()
+            return db_config
+    except IOError:
+        return f"**Could not read '/etc/postgresql/{version}/main/postgresql.conf'**"
+
+
+def report_system_information(config: Configuration) -> None:
+    """Generate a report about the host system including software versions, memory,
+    storage, and database configuration."""
+
+    with connect(make_dsn(config.get_libpq_dsn(), dbname='postgres')) as conn:
+        postgresql_ver: str = convert_version(conn.server_version_tuple())
+
+        with conn.cursor() as cur:
+            cur.execute(f"""
+            SELECT datname FROM pg_catalog.pg_database 
+            WHERE datname='{parse_dsn(config.get_libpq_dsn())['dbname']}'""")
+            nominatim_db_exists = cast(Optional[DictCursorResults], cur.fetchall())
+            if nominatim_db_exists:
+                with connect(config.get_libpq_dsn()) as conn:
+                    postgis_ver: str = convert_version(conn.postgis_version_tuple())
+            else:
+                postgis_ver = "Unable to connect to database"
+
+    postgresql_config: str = get_postgresql_config(int(float(postgresql_ver)))
+
+    # Note: psutil.disk_partitions() is similar to run_command("lsblk")
+
+    # Note: run_command("systemd-detect-virt") only works on Linux, on other OSes
+    # should give a message: "Unknown (unable to find the 'systemd-detect-virt' command)"
+
+    # Generates the Markdown report.
+
+    report = f"""
+    **Instructions**
+    Use this information in your issue report at https://github.com/osm-search/Nominatim/issues
+    Redirect the output to a file:
+    $ ./collect_os_info.py > report.md
+
+
+    **Software Environment:**
+    - Python version: {sys.version}
+    - Nominatim version: {version_str()} 
+    - PostgreSQL version: {postgresql_ver} 
+    - PostGIS version: {postgis_ver}
+    - OS: {os_name_info()}
+    
+    
+    **Hardware Configuration:**
+    - RAM: {friendly_memory_string(psutil.virtual_memory().total)}
+    - number of CPUs: {psutil.cpu_count(logical=False)}
+    - bare metal/AWS/other cloud service (per systemd-detect-virt(1)): {run_command("systemd-detect-virt")} 
+    - type and size of disks:
+    **`df -h` - df - report file system disk space usage: **
+    ```
+    {run_command(["df", "-h"])}
+    ```
+    
+    **lsblk - list block devices: **
+    ```
+    {run_command("lsblk")}
+    ```
+    
+    
+    **Postgresql Configuration:**
+    ```
+    {postgresql_config}
+    ```
+    **Notes**
+    Please add any notes about anything above anything above that is incorrect.
+"""
+    print(report)

nominatim/tools/database_import.py

@@ -75,6 +75,11 @@ def setup_database_skeleton(dsn: str, rouser: Optional[str] = None) -> None:
         with conn.cursor() as cur:
             cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')
             cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')
+
+            postgis_version = conn.postgis_version_tuple()
+            if postgis_version[0] >= 3:
+                cur.execute('CREATE EXTENSION IF NOT EXISTS postgis_raster')
+
         conn.commit()
 
         _require_version('PostGIS',
@@ -225,7 +230,8 @@ def load_data(dsn: str, threads: int) -> None:
             cur.execute('ANALYSE')
 
 
-def create_search_indices(conn: Connection, config: Configuration, drop: bool = False) -> None:
+def create_search_indices(conn: Connection, config: Configuration,
+                          drop: bool = False, threads: int = 1) -> None:
     """ Create tables that have explicit partitioning.
     """
 
@@ -243,4 +249,5 @@ def create_search_indices(conn: Connection, config: Configuration, drop: bool =
 
     sql = SQLPreprocessor(conn, config)
 
-    sql.run_sql_file(conn, 'indices.sql', drop=drop)
+    sql.run_parallel_sql_file(config.get_libpq_dsn(),
+                              'indices.sql', min(8, threads), drop=drop)

nominatim/tools/exec_utils.py

@@ -10,6 +10,7 @@ Helper functions for executing external programs.
 from typing import Any, Union, Optional, Mapping, IO
 from pathlib import Path
 import logging
+import os
 import subprocess
 import urllib.request as urlrequest
 from urllib.parse import urlencode
@@ -116,21 +117,27 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
     env = get_pg_env(options['dsn'])
     cmd = [str(options['osm2pgsql']),
            '--hstore', '--latlon', '--slim',
-           '--with-forward-dependencies', 'false',
            '--log-progress', 'true',
            '--number-processes', str(options['threads']),
            '--cache', str(options['osm2pgsql_cache']),
-           '--output', 'gazetteer',
            '--style', str(options['osm2pgsql_style'])
           ]
-    if options['append']:
+
-        cmd.append('--append')
+    if str(options['osm2pgsql_style']).endswith('.lua'):
+        env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / 'flex-base.lua'),
+                                    os.environ.get('LUAPATH', ';')))
+        cmd.extend(('--output', 'flex'))
     else:
-        cmd.append('--create')
+        cmd.extend(('--output', 'gazetteer'))
+
+    cmd.append('--append' if options['append'] else '--create')
 
     if options['flatnode_file']:
         cmd.extend(('--flat-nodes', options['flatnode_file']))
 
+    if not options.get('forward_dependencies', False):
+        cmd.extend(('--with-forward-dependencies', 'false'))
+
     for key, param in (('slim_data', '--tablespace-slim-data'),
                        ('slim_index', '--tablespace-slim-index'),
                        ('main_data', '--tablespace-main-data'),

nominatim/tools/migration.py

@@ -315,3 +315,36 @@ def mark_internal_country_names(conn: Connection, config: Configuration, **_: An
                     names = {}
                 names['countrycode'] = country_code
                 analyzer.add_country_names(country_code, names)
+
+
+@_migration(4, 1, 99, 0)
+def add_place_deletion_todo_table(conn: Connection, **_: Any) -> None:
+    """ Add helper table for deleting data on updates.
+
+        The table is only necessary when updates are possible, i.e.
+        the database is not in freeze mode.
+    """
+    if conn.table_exists('place'):
+        with conn.cursor() as cur:
+            cur.execute("""CREATE TABLE IF NOT EXISTS place_to_be_deleted (
+                             osm_type CHAR(1),
+                             osm_id BIGINT,
+                             class TEXT,
+                             type TEXT,
+                             deferred BOOLEAN)""")
+
+
+@_migration(4, 1, 99, 1)
+def split_pending_index(conn: Connection, **_: Any) -> None:
+    """ Reorganise indexes for pending updates.
+    """
+    if conn.table_exists('place'):
+        with conn.cursor() as cur:
+            cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_rank_address_sector
+                           ON placex USING BTREE (rank_address, geometry_sector)
+                           WHERE indexed_status > 0""")
+            cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_rank_boundaries_sector
+                           ON placex USING BTREE (rank_search, geometry_sector)
+                           WHERE class = 'boundary' and type = 'administrative'
+                                 and indexed_status > 0""")
+            cur.execute("DROP INDEX IF EXISTS idx_placex_pendingsector")

nominatim/tools/refresh.py

@@ -15,7 +15,7 @@ from pathlib import Path
 from psycopg2 import sql as pysql
 
 from nominatim.config import Configuration
-from nominatim.db.connection import Connection
+from nominatim.db.connection import Connection, connect
 from nominatim.db.utils import execute_file
 from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.version import version_str
@@ -146,6 +146,25 @@ def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = F
 
     return 0
 
+def import_secondary_importance(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:
+    """ Replaces the secondary importance raster data table with new data.
+
+        Returns 0 if all was well and 1 if the raster SQL file could not
+        be found. Throws an exception if there was an error reading the file.
+    """
+    datafile = data_path / 'secondary_importance.sql.gz'
+    if not datafile.exists():
+        return 1
+
+    with connect(dsn) as conn:
+        postgis_version = conn.postgis_version_tuple()
+        if postgis_version[0] < 3:
+            LOG.error('PostGIS version is too old for using OSM raster data.')
+            return 2
+
+    execute_file(dsn, datafile, ignore_errors=ignore_errors)
+
+    return 0
 
 def recompute_importance(conn: Connection) -> None:
     """ Recompute wikipedia links and importance for all entries in placex.
@@ -157,7 +176,7 @@ def recompute_importance(conn: Connection) -> None:
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance
-                FROM compute_importance(extratags, country_code, osm_type, osm_id))
+                FROM compute_importance(extratags, country_code, osm_type, osm_id, centroid))
             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance

nominatim/tools/replication.py

@@ -130,10 +130,7 @@ def update(conn: Connection, options: MutableMapping[str, Any],
         if endseq is None:
             return UpdateState.NO_CHANGES
 
-        # Consume updates with osm2pgsql.
+        run_osm2pgsql_updates(conn, options)
-        options['append'] = True
-        options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
-        run_osm2pgsql(options)
 
         # Write the current status to the file
         endstate = repl.get_state_info(endseq)
@@ -143,6 +140,25 @@ def update(conn: Connection, options: MutableMapping[str, Any],
     return UpdateState.UP_TO_DATE
 
 
+def run_osm2pgsql_updates(conn: Connection, options: MutableMapping[str, Any]) -> None:
+    """ Run osm2pgsql in append mode.
+    """
+    # Remove any stale deletion marks.
+    with conn.cursor() as cur:
+        cur.execute('TRUNCATE place_to_be_deleted')
+    conn.commit()
+
+    # Consume updates with osm2pgsql.
+    options['append'] = True
+    options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
+    run_osm2pgsql(options)
+
+    # Handle deletions
+    with conn.cursor() as cur:
+        cur.execute('SELECT flush_deleted_places()')
+    conn.commit()
+
+
 def _make_replication_server(url: str, timeout: int) -> ContextManager[ReplicationServer]:
     """ Returns a ReplicationServer in form of a context manager.
 

nominatim/version.py

@@ -25,7 +25,7 @@ from typing import Optional, Tuple
 # patch level when cherry-picking the commit with the migration.
 #
 # Released versions always have a database patch level of 0.
-NOMINATIM_VERSION = (4, 1, 2, 0)
+NOMINATIM_VERSION = (4, 2, 1, 0)
 
 POSTGRESQL_REQUIRED_VERSION = (9, 6)
 POSTGIS_REQUIRED_VERSION = (2, 2)

settings/country-names/an.yaml

@@ -1,71 +0,0 @@
-name: 
-    default: De Nederlandse Antillen
-    af: Nederlandse Antille
-    an: Antillas Neerlandesas
-    ar: جزر الأنتيل
-    be: Нідэрландскія Антылы
-    bg: Холандски Антили
-    br: Antilhez Nederlandat
-    bs: Holandski Antili
-    ca: Antilles Neerlandeses
-    cs: Nizozemské Antily
-    cy: Antilles yr Iseldiroedd
-    da: Nederlandske Antiller
-    de: Niederländische Antillen
-    dv: ނެދަލޭންޑު އެންޓިލޭ
-    el: Ολλανδικές Αντίλλες
-    en: Netherlands Antilles
-    eo: Nederlandaj Antiloj
-    es: Antillas Neerlandesas;Antillas Holandesas;Indias Occidentales Holandesas
-    et: Hollandi Antillid
-    eu: Holandarren Antillak
-    fa: آنتیل هلند
-    fi: Alankomaiden Antillit
-    fo: Niðurlendsku Antillurnar
-    fr: Antilles néerlandaises
-    fy: Nederlânske Antillen
-    ga: Aintillí na hÍsiltíre
-    gl: Antillas Neerlandesas
-    he: האנטילים ההולנדיים
-    hi: नीदरलैंड एंटीलीज़
-    hr: Nizozemski Antili
-    hu: Holland Antillák
-    ia: Antillas Nederlandese
-    id: Antillen Belanda
-    io: Nederlandana Antili
-    is: Hollensku Antillaeyjar
-    it: Antille Olandesi
-    ja: オランダ領アンティル
-    jv: Antillen Walanda
-    ka: ნიდერლანდის ანტილები
-    kk: Антийлер
-    ko: 네덜란드령 안틸레스
-    kw: Antillys Iseldiryek
-    la: Antillae Nederlandiae
-    lb: Hollännesch Antillen
-    li: Nederlandse Antille
-    ln: Antiya ya Holanda
-    lt: Nyderlandų Antilai
-    lv: Antiļas
-    mn: Нидерландын Антиллийн Арлууд
-    mr: नेदरलँड्स अँटिल्स
-    ms: Antillen Belanda
-    nn: Dei nederlandske Antillane
-    "no": De nederlandske Antillene
-    pl: Antyle Holenderskie
-    pt: Antilhas Holandesas
-    ro: Antilele Olandeze
-    ru: Нидерландские Антилы
-    sh: Nizozemski Antili
-    sk: Holandské Antily
-    sl: Nizozemski Antili
-    sr: Холандски Антили
-    sv: Nederländska Antillerna
-    sw: Antili za Kiholanzi
-    ta: நெதர்லாந்து அண்டிலிசு
-    tg: Антил Ҳоланд
-    th: เนเธอร์แลนด์แอนทิลลิส
-    tr: Hollanda Antilleri
-    uk: Нідерландські Антильські острови
-    vi: Antille thuộc Hà Lan
-    zh: 荷属安的列斯

settings/country-names/aq.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Antarctica

settings/country-names/as.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: American Samoa

settings/country-names/aw.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Aruba

settings/country-names/ax.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Aland Islands

settings/country-names/bl.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Saint Barthélemy

settings/country-names/bq.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: "\N"

settings/country-names/bv.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Bouvet Island

settings/country-names/cc.yaml

@@ -1,37 +0,0 @@
-name: 
-    default: Cocos (Keeling) Islands
-    af: Cocos (Keeling) Eilande
-    ar: جزر كوكوس (كيلينغ)
-    be: Какосавыя (Кілінг) астравы
-    br: Inizi Kokoz
-    ca: Illes Cocos
-    da: Cocosøerne
-    de: Kokosinseln
-    el: Νησιά Κόκος
-    en: Cocos (Keeling) Islands
-    eo: Kokosinsuloj
-    es: Islas Cocos (Keeling)
-    et: Kookossaared
-    eu: Cocos (Keeling) uharteak
-    fa: جزایر کوکوس
-    fi: Kookossaaret
-    fr: Îles Cocos
-    fy: de Kokoseilannen
-    he: איי קוקוס (קילינג)
-    hr: Kokosovi otoci
-    hu: Kókusz (Keeling)-szigetek
-    id: Kepulauan Cocos (Keeling)
-    is: Kókoseyjar
-    it: Isole Cocos e Keeling
-    lt: Kokoso (Keelingo) salos
-    lv: Kokosu (Kīlinga) salas
-    mn: Кокосын (Кийлингийн) Арлууд
-    nl: Cocoseilanden
-    pl: Wyspy Kokosowe
-    ru: Кокосовые острова
-    sl: Kokosovi otoki
-    sv: Kokosöarna
-    tr: Cocos (Keeling) Adaları
-    uk: Кокосові острови
-    vi: Quần đảo Cocos (Keeling)
-    zh: 科科斯（基林）群島

settings/country-names/cw.yaml

@@ -1,7 +0,0 @@
-name: 
-    default: Curaçao
-    en: Curaçao
-    es: Curazao
-    fr: Curaçao
-    ru: Кюрасао
-    sv: Curaçao

settings/country-names/cx.yaml

@@ -1,61 +0,0 @@
-name: 
-    default: Christmas Island
-    af: Christmas-eiland
-    ar: جزيرة الميلاد
-    bg: Рождество
-    br: Enez Nedeleg
-    bs: Božićno ostrvo
-    ca: Illa Christmas
-    cs: Vánoční ostrov
-    cy: Ynys y Nadolig
-    da: Juleøen
-    de: Weihnachtsinsel
-    el: Νήσος των Χριστουγέννων
-    eo: Kristnaskinsulo
-    es: Isla de Navidad
-    et: Jõulusaar
-    eu: Christmas uhartea
-    fa: جزیره کریسمس
-    fi: Joulusaari
-    fr: Île Christmas
-    fy: Krysteilân
-    ga: Oileán na Nollag
-    gl: Illa de Nadal
-    he: טריטוריית האי חג המולד
-    hi: क्रिसमस आईलैंड
-    hr: Božićni otok
-    hu: Karácsony-sziget
-    id: Pulau Natal
-    is: Jólaeyja
-    it: Isola del Natale
-    ja: クリスマス島
-    ka: შობის კუნძული
-    kk: Кристмас аралы
-    ko: 크리스마스 섬
-    kw: Ynys Nadelik
-    lb: Chrëschtdagsinsel
-    lt: Kalėdų sala
-    lv: Ziemsvētku sala
-    mn: Зул Сарын Арал
-    mr: क्रिसमस द्वीप
-    ms: Pulau Krismas
-    nl: Christmaseiland
-    nn: Christmasøya
-    "no": Christmasøya
-    pl: Wyspa Bożego Narodzenia
-    pt: Ilha Christmas
-    ro: Insula Crăciunului
-    ru: Остров Рождества
-    sh: Božićni otok
-    sk: Vianočný ostrov
-    sl: Božični otoki
-    sr: Божићно Острво
-    sv: Julön
-    sw: Kisiwa cha Krismasi
-    ta: கிறிஸ்துமசு தீவு
-    th: เกาะคริสต์มาส
-    tr: Christmas Adası
-    uk: Острів Різдва
-    vi: Đảo Christmas
-    wo: Dunu Christmas
-    zh: 圣诞岛

settings/country-names/gf.yaml

@@ -1,41 +0,0 @@
-name: 
-    default: Guyane Française
-    af: Frans-Guyana
-    ar: غيانا
-    br: Gwiana c’hall
-    ca: Guaiana Francesa
-    cy: Guyane
-    da: Fransk Guyana
-    de: Französisch-Guayana
-    el: Γαλλική Γουιάνα
-    en: French Guiana
-    eo: Gujano
-    es: Guayana Francesa
-    et: Prantsuse Guajaana
-    fa: گویان فرانسه
-    fi: Ranskan Guayana
-    fr: Guyane française
-    fy: Frânsk Guyana
-    ga: Guáin na Fraince
-    gd: Guiana Fhrangach
-    he: גיאנה הצרפתית
-    hr: Francuska Gvajana
-    hu: Francia Guyana
-    id: Guyana Perancis
-    is: Franska Gvæjana
-    it: Guyana francese
-    la: Guiana Francica
-    li: Frans Guyana
-    lt: Prancūzijos Gviana
-    lv: Franču Gviāna
-    mn: Франц Гвиана
-    nl: Frans-Guyana
-    pl: Gujana Francuska
-    ru: Французская Гвиана
-    sl: Francoska Gvajana
-    sv: Franska Guyana
-    th: เฟรนช์เกียนา
-    tr: Fransız Guyanası
-    uk: Французька Гвіана
-    vi: Guyane thuộc Pháp
-    zh: 法属圭亚那

settings/country-names/gp.yaml

@@ -1,31 +0,0 @@
-name: 
-    default: Guadeloupe
-    ar: غوادلوب
-    be: Гвадэлупа
-    br: Gwadeloup
-    ca: Illa de Guadalupe
-    da: Guadeloupe
-    el: Γουαδελούπη
-    en: Guadeloupe
-    eo: Gvadelupo
-    es: Guadalupe
-    fa: گوادلوپ
-    fi: Guadeloupe
-    fr: Guadeloupe
-    fy: Guadelûp
-    ga: Guadalúip
-    he: גוואדלופ
-    hr: Gvadalupa
-    hu: Guadeloupe
-    is: Gvadelúpeyjar
-    it: Guadalupa
-    la: Guadalupa
-    lt: Gvadelupa
-    lv: Gvadelupa
-    mn: Гуаделупе
-    pl: Gwadelupa
-    ru: Гваделупа
-    sv: Guadeloupe
-    th: กวาเดอลูป
-    uk: Гваделупа
-    zh: 瓜德罗普

settings/country-names/gu.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Guam

settings/country-names/hk.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Hong Kong

settings/country-names/hm.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Heard Island and MaxDonald Islands

settings/country-names/mf.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Saint Martin

settings/country-names/mo.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Macao

settings/country-names/mp.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Northern Mariana Islands

settings/country-names/mq.yaml

@@ -1,30 +0,0 @@
-name: 
-    default: Martinique
-    ar: مارتينيك
-    be: Марцініка
-    br: Martinik
-    ca: Martinica
-    da: Martinique
-    el: Μαρτινίκα
-    en: Martinique
-    eo: Martiniko
-    es: Martinica
-    fa: مارتینیک
-    fi: Martinique
-    fr: Martinique
-    fy: Martinyk
-    he: מרטיניק
-    hr: Martinik
-    hu: Martinique
-    id: Martinik
-    is: Martinique
-    it: Martinica
-    la: Martinica
-    lt: Martinika
-    lv: Martinika
-    mn: Мартиник
-    pl: Martynika
-    ru: Мартиника
-    sv: Martinique
-    uk: Мартиніка
-    zh: 馬提尼克

settings/country-names/nc.yaml

@@ -1,37 +0,0 @@
-name: 
-    default: Nouvelle-Calédonie
-    af: Nieu-Caledonia
-    ar: كاليدونيا الجديدة
-    be: Новая Каледонія
-    br: Kaledonia Nevez
-    ca: Nova Caledònia
-    cy: Caledonia Newydd
-    da: Ny Kaledonien
-    de: Neukaledonien
-    el: Νέα Καληδονία
-    en: New Caledonia
-    eo: Nov-Kaledonio
-    es: Nueva Caledonia
-    fa: کالدونیای جدید
-    fi: Uusi-Kaledonia
-    fr: Nouvelle-Calédonie
-    ga: An Nua-Chaladóin
-    he: קלדוניה החדשה
-    hr: Nova Kaledonija
-    hu: Új-Kaledónia
-    id: Kaledonia Baru
-    is: Nýja-Kaledónía
-    it: Nuova Caledonia
-    la: Nova Caledonia
-    lt: Naujoji Kaledonija
-    lv: Jaunkaledonija
-    mn: Шинэ Каледони
-    nl: Nieuw-Caledonië
-    pl: Nowa Kaledonia
-    ru: Новая Каледония
-    sl: Nova Kaledonija
-    sv: Nya Kaledonien
-    th: นิวแคลิโดเนีย
-    tr: Yeni Kaledonya
-    uk: Нова Каледонія
-    zh: 新喀里多尼亚

settings/country-names/nf.yaml

@@ -1,36 +0,0 @@
-name: 
-    default: Norfolk Island
-    af: Norfolkeiland
-    ar: جزيرة نورفولك
-    be: Норфалк
-    br: Enez Norfolk
-    ca: Illa Norfolk
-    cy: Ynys Norfolk
-    da: Norfolk-øen
-    de: Norfolkinsel
-    en: Norfolk Island
-    eo: Norfolkinsulo
-    es: Isla Norfolk
-    et: Norfolki saar
-    fi: Norfolkinsaari
-    fr: Île Norfolk
-    fy: Norfolk
-    ga: Oileán Norfolk
-    he: האי נורפוק
-    hr: Otok Norfolk
-    hu: Norfolk-sziget
-    id: Pulau Norfolk
-    is: Norfolkeyja
-    it: Isola Norfolk
-    la: Insula Norfolcia
-    lt: Norfolko sala
-    lv: Norfolkas sala
-    mn: Норфолк Арал
-    nl: Norfolk
-    pl: Wyspa Norfolk
-    ru: Остров Норфолк
-    sv: Norfolkön
-    tr: Norfolk Adası
-    uk: Острів Норфолк
-    vi: Đảo Norfolk
-    zh: 诺福克岛

settings/country-names/pf.yaml

@@ -1,77 +0,0 @@
-name: 
-    default: Polynésie française
-    af: Franse Polynesië
-    an: Polinesia Franzesa
-    ar: بولونيزيا الفرنسية
-    az: Fransa Polineziyası
-    be: Французская Палінезія
-    bg: Френска Полинезия
-    br: Polinezia Frañs
-    bs: Francuska Polinezija
-    ca: Polinèsia Francesa
-    cs: Francouzská Polynésie
-    cy: Polynesia Ffrengig
-    da: Fransk Polynesien
-    de: Französisch-Polynesien
-    dv: ފަރަންސޭސި ޕޮލިނޭޝިއާ
-    el: Γαλλική Πολυνησία
-    en: French Polynesia
-    eo: Franca Polinezio
-    es: Polinesia Francesa
-    et: Prantsuse Polüneesia
-    eu: Frantziar Polinesia
-    fa: پلی‌نزی فرانسه
-    fi: Ranskan Polynesia
-    fr: Polynésie française
-    fy: Frânsk Polyneezje
-    ga: Polainéis na Fraince
-    gd: French Polynesia
-    gl: Polinesia francesa
-    he: פולינזיה הצרפתית
-    hi: फ्रेंच पोलीनेशिया
-    hr: Francuska Polinezija
-    hu: Francia Polinézia
-    id: Polinesia Perancis
-    io: Franca Polinezia
-    is: Franska Pólýnesía
-    it: Polinesia francese
-    ja: フランス領ポリネシア
-    jv: Polinesia Perancis
-    kk: Франция Полинезиясы
-    ko: 프랑스령 폴리네시아
-    kw: Polynesi Frynkek
-    la: Polynesia Francica
-    lb: Franséisch-Polynesien
-    lt: Prancūzijos Polinezija
-    lv: Franču Polinēzija
-    mi: Porinīhia Wīwī
-    mk: Француска Полинезија
-    mn: Францын Полинез
-    mr: फ्रेंच पॉलिनेशिया
-    ms: Polinesia Perancis
-    nl: Frans-Polynesië
-    nn: Fransk Polynesia
-    "no": Fransk Polynesia
-    oc: Polinesia Francesa
-    os: Францы Полинези
-    pl: Polinezja Francuska
-    pt: Polinésia Francesa
-    qu: Phransis Pulinisya
-    ro: Polinezia Franceză
-    ru: Французская Полинезия
-    se: Frankriikka Polynesia
-    sh: Francuska Polinezija
-    sk: Francúzska Polynézia
-    sl: Francoska Polinezija
-    sr: Француска Полинезија
-    sv: Franska Polynesien
-    sw: Polynesia ya Kifaransa
-    ta: பிரெஞ்சு பொலினீசியா
-    th: เฟรนช์โปลินีเซีย
-    tr: Fransız Polinezyası
-    ty: Pōrīnetia Farāni
-    ug: Fransiyige Qarashliq Polinéziye
-    uk: Французька Полінезія
-    vi: Polynésie thuộc Pháp
-    wo: Polineesi gu Faraas
-    zh: 法属波利尼西亚

settings/country-names/pm.yaml

@@ -1,19 +0,0 @@
-name: 
-    default: Saint-Pierre-et-Miquelon
-    af: Saint-Pierre et Miquelon
-    be: Святы П’ер і Міквелон
-    da: Saint Pierre og Miquelon
-    de: Saint-Pierre und Miquelon
-    en: Saint Pierre and Miquelon
-    eo: Sankta-Piero kaj Mikelono
-    es: San Pedro y Miguelón
-    fi: Saint-Pierre ja Miquelon
-    fr: Saint-Pierre-et-Miquelon
-    hr: Sveti Petar i Mikelon
-    hu: Saint-Pierre és Miquelon
-    lt: Sen Pjeras ir Mikelonas
-    lv: Senpjēra un Mikelona
-    mn: Сент Пьер ба Микелон
-    sv: Saint-Pierre och Miquelon
-    tr: Saint-Pierre ve Miquelon
-    uk: Сен-П'єр і Мікелон

settings/country-names/pr.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Puerto Rico

settings/country-names/re.yaml

@@ -1,29 +0,0 @@
-name: 
-    default: Réunion
-    af: Réunion
-    ar: ريونيون
-    be: Руньён
-    br: Ar Reunion
-    ca: Illa de la Reunió
-    da: Reunion
-    el: Ρεϊνιόν
-    eo: Reunio
-    es: La Reunión
-    fa: رئونیون
-    fi: Réunion
-    fr: La Réunion
-    he: ראוניון
-    hu: Réunion
-    is: Réunion
-    it: Riunione
-    la: Reunio
-    lt: Reunionas
-    lv: Reinjona
-    mn: Реюньон
-    pl: Reunion
-    ru: Реюньон
-    sl: Reunion
-    sv: Réunion
-    th: เรอูนียง
-    uk: Реюньйон
-    zh: 留尼汪

settings/country-names/sj.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Svalbard and Jan Mayen

settings/country-names/sx.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Sint Maarten

settings/country-names/tf.yaml

@@ -1,48 +0,0 @@
-name: 
-    default: Terres australes et antarctiques françaises
-    af: Franse Suidelike en Antarktiese Gebiede
-    an: Territorios Australs Franzeses
-    ar: الأراضي الجنوبية الفرنسية
-    be: Французскія Паўднёвыя тэрыторыі
-    bg: Френски южни и антарктически територии
-    br: Douaroù Aostral hag Antarktikel Frañs
-    ca: Terres Australs i Antàrtiques Franceses
-    cs: Francouzská jižní a antarktická území
-    da: Franske sydlige og Antarktiske territorier
-    de: Französische Süd- und Antarktisgebiete
-    el: Γαλλικά νότια και ανταρκτικά εδάφη
-    en: French Southern Lands
-    eo: Francaj Sudaj Teritorioj
-    es: Tierras Australes y Antárticas Francesas
-    eu: Frantziaren lurralde austral eta antartikoak
-    fi: Ranskan eteläiset ja antarktiset alueet
-    fr: Terres australes et antarctiques françaises
-    fy: Frânske Súdlike en Antarktyske Lannen
-    gl: Terras Austrais e Antárticas Francesas
-    hr: Francuski južni i antarktički teritoriji
-    hu: Francia déli és antarktiszi területek
-    id: Daratan Selatan dan Antarktika Perancis
-    is: Frönsku suðlægu landsvæðin
-    it: Terre Australi e Antartiche Francesi
-    ja: フランス領南方・南極地域
-    ko: 프랑스령 남부와 남극 지역
-    kw: Tiryow Deghow hag Antarktik Frynkek
-    lt: Prancūzijos Pietų Sritys
-    lv: Francijas Dienvidjūru un Antarktikas Zemes
-    nl: Franse Zuidelijke en Antarctische Gebieden
-    "no": De franske sørterritorier
-    oc: Tèrras Australas e Antarticas Francesas
-    pl: Francuskie Terytoria Południowe i Antarktyczne
-    pt: Terras Austrais e Antárticas Francesas
-    ro: Teritoriile australe şi antarctice franceze
-    ru: Французские Южные и Антарктические территории
-    sh: Francuske Južne Teritorije
-    sk: Francúzske južné a antarktické územia
-    sl: Francoske južne in antarktične dežele
-    sr: Француске јужне и антарктичке земље
-    sv: Franska sydterritorierna
-    ta: பிரெஞ்சு தென்னக நிலங்களும் அண்டாடிக் நிலமும்
-    tr: Fransız Güney ve Antarktika Toprakları
-    uk: Французькі Південні та Антарктичні території
-    vi: Vùng đất phía Nam và châu Nam Cực thuộc Pháp
-    zh: 法属南部领地

settings/country-names/um.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: United States Minor Outlying Islands

settings/country-names/vi.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: United States Virgin Islands

settings/country-names/wf.yaml

@@ -1,68 +0,0 @@
-name: 
-    default: Wallis-et-Futuna
-    af: Wallis-en-Futuna
-    an: Wallis e Futuna
-    ar: جزر واليس وفوتونا
-    be: Уоліс і Футуна
-    bg: Уолис и Футуна
-    br: Wallis ha Futuna
-    ca: Wallis i Futuna
-    cs: Wallis a Futuna
-    cy: Wallis a Futuna
-    da: Wallis og Futuna
-    de: Wallis und Futuna
-    dv: ވާލީ އަދި ފުތޫނާ
-    el: Ουώλλις και Φουτούνα
-    en: Wallis and Futuna Islands
-    eo: Valiso kaj Futuno
-    es: Wallis y Futuna
-    et: Wallis ja Futuna
-    eu: Wallis eta Futuna
-    fa: والیس و فوتونا
-    fi: Wallis- ja Futunasaaret
-    fr: Wallis-et-Futuna
-    fy: Wallis en Fûtûna
-    ga: Vailís agus Futúna
-    gl: Wallis e Futuna
-    he: ואליס ופוטונה
-    hr: Wallis i Futuna
-    hu: Wallis és Futuna
-    id: Wallis dan Futuna
-    io: Wallis e Futuna Insuli
-    is: Wallis- og Fútúnaeyjar
-    it: Wallis e Futuna
-    ja: ウォリス・フツナ
-    jv: Wallis lan Futuna
-    ko: 왈리스 퓌튀나
-    kw: Wallis ha Futuna
-    la: Vallis et Futuna
-    lb: Wallis a Futuna
-    lt: Walliso ir Futuna salos
-    lv: Volisa un Futuna
-    mn: Уоллис ба Футуна
-    mr: वालिस व फुतुना
-    ms: Wallis dan Futuna
-    nl: Wallis en Futuna
-    nn: Wallis- og Futunaøyane
-    "no": Wallis- og Futunaøyene
-    oc: Wallis e Futuna
-    pl: Wallis i Futuna
-    pt: Wallis e Futuna
-    ro: Wallis şi Futuna
-    ru: Уоллис и Футуна
-    se: Wallis ja Futuna
-    sh: Wallis i Futuna
-    sk: Wallis a Futuna
-    sl: Wallis in Futuna
-    sm: Wallis and Futuna
-    sr: Валис и Футуна
-    sv: Wallis- och Futunaöarna
-    sw: Wallis na Futuna
-    ta: வலிசும் புட்டூனாவும்
-    th: หมู่เกาะวาลลิสและหมู่เกาะฟุตูนา
-    tr: Wallis ve Futuna Adaları
-    ug: Wallis we Futuna Taqim Aralliri
-    uk: Волліс і Футуна
-    vi: Wallis và Futuna
-    wo: Wallis ak Futuna
-    zh: 瓦利斯和富图纳群岛

settings/country-names/yt.yaml

@@ -1,2 +0,0 @@
-name: 
-    default: Mayotte

settings/country_settings.yaml

@@ -61,13 +61,6 @@ am:
       pattern: "dddd"
 
 
-# Netherlands Antilles (De Nederlandse Antillen)
-an:
-    partition: 58
-    languages: nl, en, pap
-    names: !include country-names/an.yaml
-
-
 # Angola (Angola)
 ao:
     partition: 85
@@ -76,14 +69,6 @@ ao:
     postcode: no
 
 
-#  (Antarctica)
-aq:
-    partition: 181
-    languages: en, es, fr, ru
-    names: !include country-names/aq.yaml
-    postcode: no
-
-
 # Argentina (Argentina)
 ar:
     partition: 39
@@ -93,13 +78,6 @@ ar:
       pattern: "l?dddd(?:lll)?"
 
 
-#  (American Samoa)
-as:
-    partition: 182
-    languages: en, sm
-    names: !include country-names/as.yaml
-
-
 # Austria (Österreich)
 at:
     partition: 245
@@ -118,21 +96,6 @@ au:
       pattern: "dddd"
 
 
-#  (Aruba)
-aw:
-    partition: 183
-    languages: nl, pap
-    names: !include country-names/aw.yaml
-    postcode: no
-
-
-#  (Aland Islands)
-ax:
-    partition: 184
-    languages: sv
-    names: !include country-names/ax.yaml
-
-
 # Azerbaijan (Azərbaycan)
 az:
     partition: 119
@@ -221,13 +184,6 @@ bj:
     postcode: no
 
 
-#  (Saint Barthélemy)
-bl:
-    partition: 204
-    languages: fr
-    names: !include country-names/bl.yaml
-
-
 # Bermuda (Bermuda)
 bm:
     partition: 176
@@ -256,13 +212,6 @@ bo:
     postcode: no
 
 
-# Caribbean Netherlands (Caribisch Nederland)
-bq:
-    partition: 250
-    languages: nl
-    names: !include country-names/bq.yaml
-
-
 # Brazil (Brasil)
 br:
     partition: 121
@@ -290,13 +239,6 @@ bt:
       pattern: "ddddd"
 
 
-#  (Bouvet Island)
-bv:
-    partition: 185
-    languages: "no"
-    names: !include country-names/bv.yaml
-
-
 # Botswana (Botswana)
 bw:
     partition: 122
@@ -332,13 +274,6 @@ ca:
       output: \1 \2
 
 
-# Cocos (Keeling) Islands (Cocos (Keeling) Islands)
-cc:
-    partition: 118
-    languages: en
-    names: !include country-names/cc.yaml
-
-
 # Democratic Republic of the Congo (République démocratique du Congo)
 cd:
     partition: 229
@@ -450,20 +385,6 @@ cv:
       pattern: "dddd"
 
 
-# Curaçao (Curaçao)
-cw:
-    partition: 248
-    languages: nl, en
-    names: !include country-names/cw.yaml
-
-
-# Christmas Island (Christmas Island)
-cx:
-    partition: 177
-    languages: en
-    names: !include country-names/cx.yaml
-
-
 # Cyprus (Κύπρος - Kıbrıs)
 cy:
     partition: 114
@@ -683,13 +604,6 @@ ge:
       pattern: "dddd"
 
 
-# French Guiana (Guyane Française)
-gf:
-    partition: 231
-    languages: fr
-    names: !include country-names/gf.yaml
-
-
 # Guernsey (Guernsey)
 gg:
     partition: 77
@@ -745,13 +659,6 @@ gn:
       pattern: "ddd"
 
 
-# Guadeloupe (Guadeloupe)
-gp:
-    partition: 232
-    languages: fr
-    names: !include country-names/gp.yaml
-
-
 # Equatorial Guinea (Guinea Ecuatorial)
 gq:
     partition: 12
@@ -789,13 +696,6 @@ gt:
       pattern: "ddddd"
 
 
-# Guam (Guam)
-gu:
-    partition: 187
-    languages: en, ch
-    names: !include country-names/gu.yaml
-
-
 # Guinea-Bissau (Guiné-Bissau)
 gw:
     partition: 8
@@ -813,20 +713,6 @@ gy:
     postcode: no
 
 
-#  (Hong Kong)
-hk:
-    partition: 188
-    languages: zh-hant, en
-    names: !include country-names/hk.yaml
-
-
-#  (Heard Island and MaxDonald Islands)
-hm:
-    partition: 189
-    languages: en
-    names: !include country-names/hm.yaml
-
-
 # Honduras (Honduras)
 hn:
     partition: 56
@@ -1229,13 +1115,6 @@ me:
       pattern: "ddddd"
 
 
-# Saint Martin (Saint Martin)
-mf:
-    partition: 203
-    languages: fr
-    names: !include country-names/mf.yaml
-
-
 # Madagascar (Madagasikara)
 mg:
     partition: 164
@@ -1289,28 +1168,6 @@ mn:
       pattern: "ddddd"
 
 
-# Macao (Macao)
-mo:
-    partition: 191
-    languages: zh-hant, pt
-    names: !include country-names/mo.yaml
-    postcode: no
-
-
-# Northern Mariana Islands (Northern Mariana Islands)
-mp:
-    partition: 192
-    languages: ch, en
-    names: !include country-names/mp.yaml
-
-
-# Martinique (Martinique)
-mq:
-    partition: 233
-    languages: fr
-    names: !include country-names/mq.yaml
-
-
 # Mauritania (موريتانيا)
 mr:
     partition: 149
@@ -1398,13 +1255,6 @@ na:
       pattern: "ddddd"
 
 
-# New Caledonia (Nouvelle-Calédonie)
-nc:
-    partition: 234
-    languages: fr
-    names: !include country-names/nc.yaml
-
-
 # Niger (Niger)
 ne:
     partition: 226
@@ -1414,13 +1264,6 @@ ne:
       pattern: "dddd"
 
 
-# Norfolk Island (Norfolk Island)
-nf:
-    partition: 100
-    languages: en, pih
-    names: !include country-names/nf.yaml
-
-
 # Nigeria (Nigeria)
 ng:
     partition: 218
@@ -1519,13 +1362,6 @@ pe:
       pattern: "ddddd"
 
 
-# French Polynesia (Polynésie française)
-pf:
-    partition: 202
-    languages: fr
-    names: !include country-names/pf.yaml
-
-
 # Papua New Guinea (Papua Niugini)
 pg:
     partition: 71
@@ -1563,13 +1399,6 @@ pl:
       output: \1-\2
 
 
-# Saint Pierre and Miquelon (Saint-Pierre-et-Miquelon)
-pm:
-    partition: 236
-    languages: fr
-    names: !include country-names/pm.yaml
-
-
 # Pitcairn Islands (Pitcairn Islands)
 pn:
     partition: 113
@@ -1580,13 +1409,6 @@ pn:
       output: \1 \2
 
 
-# Puerto Rico (Puerto Rico)
-pr:
-    partition: 193
-    languages: es, en
-    names: !include country-names/pr.yaml
-
-
 # Palestinian Territory (Palestinian Territory)
 ps:
     partition: 194
@@ -1631,13 +1453,6 @@ qa:
     postcode: no
 
 
-#  (Réunion)
-re:
-    partition: 235
-    languages: fr
-    names: !include country-names/re.yaml
-
-
 # Romania (România)
 ro:
     partition: 170
@@ -1745,13 +1560,6 @@ si:
       pattern: "dddd"
 
 
-#  (Svalbard and Jan Mayen)
-sj:
-    partition: 197
-    languages: "no"
-    names: !include country-names/sj.yaml
-
-
 # Slovakia (Slovensko)
 sk:
     partition: 172
@@ -1831,13 +1639,6 @@ sv:
       pattern: "dddd"
 
 
-#  (Sint Maarten)
-sx:
-    partition: 249
-    languages: nl, en
-    names: !include country-names/sx.yaml
-
-
 # Syria (سوريا)
 sy:
     partition: 104
@@ -1873,13 +1674,6 @@ td:
     postcode: no
 
 
-# French Southern Lands (Terres australes et antarctiques françaises)
-tf:
-    partition: 132
-    languages: fr
-    names: !include country-names/tf.yaml
-
-
 # Togo (Togo)
 tg:
     partition: 243
@@ -2009,15 +1803,6 @@ ug:
     postcode: no
 
 
-#  (United States Minor Outlying Islands)
-um:
-    partition: 198
-    languages: en
-    names: !include country-names/um.yaml
-    postcode:
-      pattern: "96898"
-
-
 # United States (United States)
 us:
     partition: 2
@@ -2083,13 +1868,6 @@ vg:
       output: VG\1
 
 
-#  (United States Virgin Islands)
-vi:
-    partition: 199
-    languages: en
-    names: !include country-names/vi.yaml
-
-
 # Vietnam (Việt Nam)
 vn:
     partition: 75
@@ -2107,13 +1885,6 @@ vu:
     postcode: no
 
 
-# Wallis and Futuna Islands (Wallis-et-Futuna)
-wf:
-    partition: 238
-    languages: fr
-    names: !include country-names/wf.yaml
-
-
 # Samoa (Sāmoa)
 ws:
     partition: 131
@@ -2138,13 +1909,6 @@ ye:
     postcode: no
 
 
-# Mayotte (Mayotte)
-yt:
-    partition: 200
-    languages: fr
-    names: !include country-names/yt.yaml
-
-
 # South Africa (South Africa)
 za:
     partition: 76

settings/env.defaults

@@ -33,6 +33,15 @@ NOMINATIM_MAX_WORD_FREQUENCY=50000
 # If true, admin level changes on places with many contained children are blocked.
 NOMINATIM_LIMIT_REINDEXING=yes
 
+# When set to 'yes' changes to OSM objects will be propagated to dependent
+# objects. This means that geometries of way/relations are updated when a
+# node on a way or a way in a relation changes.
+# EXPERT ONLY: Use with care. Enabling this option might lead to significantly
+#              more load when updates are applied.
+#              Do not enable this option on an existing database.
+# The default is to not propagate these changes.
+NOMINATIM_UPDATE_FORWARD_DEPENDENCIES=no
+
 # Restrict search languages.
 # Normally Nominatim will include all language variants of name:XX
 # in the search index. Set this to a comma separated list of language

settings/flex-base.lua

@@ -0,0 +1,383 @@
+-- Core functions for Nominatim import flex style.
+--
+
+
+-- The single place table.
+place_table = osm2pgsql.define_table{
+    name = "place",
+    ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
+    columns = {
+        { column = 'class', type = 'text', not_null = true },
+        { column = 'type', type = 'text', not_null = true },
+        { column = 'admin_level', type = 'smallint' },
+        { column = 'name', type = 'hstore' },
+        { column = 'address', type = 'hstore' },
+        { column = 'extratags', type = 'hstore' },
+        { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
+    }
+}
+
+------------- Place class ------------------------------------------
+
+local Place = {}
+Place.__index = Place
+
+function Place.new(object, geom_func)
+    local self = setmetatable({}, Place)
+    self.object = object
+    self.geom_func = geom_func
+
+    self.admin_level = tonumber(self.object:grab_tag('admin_level'))
+    if self.admin_level == nil
+       or self.admin_level <= 0 or self.admin_level > 15
+       or math.floor(self.admin_level) ~= self.admin_level then
+        self.admin_level = 15
+    end
+
+    self.num_entries = 0
+    self.has_name = false
+    self.names = {}
+    self.address = {}
+    self.extratags = {}
+
+    return self
+end
+
+function Place:delete(data)
+    if data.match ~= nil then
+        for k, v in pairs(self.object.tags) do
+            if data.match(k, v) then
+                self.object.tags[k] = nil
+            end
+        end
+    end
+end
+
+function Place:grab_extratags(data)
+    local count = 0
+
+    if data.match ~= nil then
+        for k, v in pairs(self.object.tags) do
+            if data.match(k, v) then
+                self.object.tags[k] = nil
+                self.extratags[k] = v
+                count = count + 1
+            end
+        end
+    end
+
+    return count
+end
+
+function Place:grab_address(data)
+    local count = 0
+
+    if data.match ~= nil then
+        for k, v in pairs(self.object.tags) do
+            if data.match(k, v) then
+                self.object.tags[k] = nil
+
+                if data.include_on_name == true then
+                    self.has_name = true
+                end
+
+                if data.out_key ~= nil then
+                    self.address[data.out_key] = v
+                    return 1
+                end
+
+                if k:sub(1, 5) == 'addr:' then
+                    self.address[k:sub(6)] = v
+                elseif k:sub(1, 6) == 'is_in:' then
+                    self.address[k:sub(7)] = v
+                else
+                    self.address[k] = v
+                end
+                count = count + 1
+            end
+        end
+    end
+
+    return count
+end
+
+function Place:set_address(key, value)
+    self.address[key] = value
+end
+
+function Place:grab_name(data)
+    local count = 0
+
+    if data.match ~= nil then
+        for k, v in pairs(self.object.tags) do
+            if data.match(k, v) then
+                self.object.tags[k] = nil
+                self.names[k] = v
+                if data.include_on_name ~= false then
+                    self.has_name = true
+                end
+                count = count + 1
+            end
+        end
+    end
+
+    return count
+end
+
+function Place:grab_tag(key)
+    return self.object:grab_tag(key)
+end
+
+function Place:tags()
+    return self.object.tags
+end
+
+function Place:write_place(k, v, mtype, save_extra_mains)
+    if mtype == nil then
+        return 0
+    end
+
+    v = v or self.object.tags[k]
+    if v == nil then
+        return 0
+    end
+
+    if type(mtype) == 'table' then
+        mtype = mtype[v] or mtype[1]
+    end
+
+    if mtype == 'always' or (self.has_name and mtype == 'named') then
+        return self:write_row(k, v, save_extra_mains)
+    end
+
+    if mtype == 'named_with_key' then
+        local names = {}
+        local prefix = k .. ':name'
+        for namek, namev in pairs(self.object.tags) do
+            if namek:sub(1, #prefix) == prefix
+               and (#namek == #prefix
+                    or namek:sub(#prefix + 1, #prefix + 1) == ':') then
+                names[namek:sub(#k + 2)] = namev
+            end
+        end
+
+        if next(names) ~= nil then
+            local saved_names = self.names
+            self.names = names
+
+            local results = self:write_row(k, v, save_extra_mains)
+
+            self.names = saved_names
+
+            return results
+        end
+    end
+
+    return 0
+end
+
+function Place:write_row(k, v, save_extra_mains)
+    if self.geometry == nil then
+        self.geometry = self.geom_func(self.object)
+    end
+    if self.geometry:is_null() then
+        return 0
+    end
+
+    if save_extra_mains then
+        for extra_k, extra_v in pairs(self.object.tags) do
+            if extra_k ~= k then
+                self.extratags[extra_k] = extra_v
+            end
+        end
+    end
+
+    place_table:insert{
+        class = k,
+        type = v,
+        admin_level = self.admin_level,
+        name = next(self.names) and self.names,
+        address = next(self.address) and self.address,
+        extratags = next(self.extratags) and self.extratags,
+        geometry = self.geometry
+    }
+
+    if save_extra_mains then
+        for k, v in pairs(self.object.tags) do
+            self.extratags[k] = nil
+        end
+    end
+
+    self.num_entries = self.num_entries + 1
+
+    return 1
+end
+
+
+function tag_match(data)
+    if data == nil or next(data) == nil then
+        return nil
+    end
+
+    local fullmatches = {}
+    local key_prefixes = {}
+    local key_suffixes = {}
+
+    if data.keys ~= nil then
+        for _, key in pairs(data.keys) do
+            if key:sub(1, 1) == '*' then
+                if #key > 1 then
+                    if key_suffixes[#key - 1] == nil then
+                        key_suffixes[#key - 1] = {}
+                    end
+                    key_suffixes[#key - 1][key:sub(2)] = true
+                end
+            elseif key:sub(#key, #key) == '*' then
+                if key_prefixes[#key - 1] == nil then
+                    key_prefixes[#key - 1] = {}
+                end
+                key_prefixes[#key - 1][key:sub(1, #key - 1)] = true
+            else
+                fullmatches[key] = true
+            end
+        end
+    end
+
+    if data.tags ~= nil then
+        for k, vlist in pairs(data.tags) do
+            if fullmatches[k] == nil then
+                fullmatches[k] = {}
+                for _, v in pairs(vlist) do
+                    fullmatches[k][v] = true
+                end
+            end
+        end
+    end
+
+    return function (k, v)
+        if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then
+            return true
+        end
+
+        for slen, slist in pairs(key_suffixes) do
+            if #k >= slen and slist[k:sub(-slen)] ~= nil then
+                return true
+            end
+        end
+
+        for slen, slist in pairs(key_prefixes) do
+            if #k >= slen and slist[k:sub(1, slen)] ~= nil then
+                return true
+            end
+        end
+
+        return false
+    end
+end
+
+
+-- Process functions for all data types
+function osm2pgsql.process_node(object)
+
+    local function geom_func(o)
+        return o:as_point()
+    end
+
+    process_tags(Place.new(object, geom_func))
+end
+
+function osm2pgsql.process_way(object)
+
+    local function geom_func(o)
+        local geom = o:as_polygon()
+
+        if geom:is_null() then
+            geom = o:as_linestring()
+        end
+
+        return geom
+    end
+
+    process_tags(Place.new(object, geom_func))
+end
+
+function relation_as_multipolygon(o)
+    return o:as_multipolygon()
+end
+
+function relation_as_multiline(o)
+    return o:as_multilinestring():line_merge()
+end
+
+function osm2pgsql.process_relation(object)
+    local geom_func = RELATION_TYPES[object.tags.type]
+
+    if geom_func ~= nil then
+        process_tags(Place.new(object, geom_func))
+    end
+end
+
+function process_tags(o)
+    local fallback
+
+    o:delete{match = PRE_DELETE}
+    o:grab_extratags{match = PRE_EXTRAS}
+
+    -- Exception for boundary/place double tagging
+    if o.object.tags.boundary == 'administrative' then
+        o:grab_extratags{match = function (k, v)
+            return k == 'place' and v:sub(1,3) ~= 'isl'
+        end}
+    end
+
+    -- address keys
+    o:grab_address{match=COUNTRY_TAGS, out_key='country'}
+    if o.address.country ~= nil and #o.address.country ~= 2 then
+        o.address['country'] = nil
+    end
+    if o:grab_name{match=HOUSENAME_TAGS} > 0 then
+        fallback = {'place', 'house'}
+    end
+    if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then
+        fallback = {'place', 'house'}
+    end
+    if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then
+        fallback = {'place', 'postcode'}
+    end
+
+    local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0
+
+    o:grab_address{match=ADDRESS_TAGS}
+
+    if is_interpolation then
+        o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
+        return
+    end
+
+    -- name keys
+    o:grab_name{match = NAMES}
+    o:grab_name{match = REFS, include_on_name = false}
+
+    o:delete{match = POST_DELETE}
+    o:grab_extratags{match = POST_EXTRAS}
+
+    -- collect main keys
+    local num_mains = 0
+    for k, v in pairs(o:tags()) do
+        num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
+    end
+
+    if num_mains == 0 then
+        for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do
+            if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then
+                return
+            end
+        end
+
+        if fallback ~= nil then
+            o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS)
+        end
+    end
+end
+
+

settings/icu_tokenizer.yaml

@@ -35,6 +35,7 @@ sanitizers:
     - step: clean-postcodes
       convert-to-address: yes
       default-pattern: "[A-Z0-9- ]{3,12}"
+    - step: clean-tiger-tags
     - step: split-name-list
     - step: strip-brace-terms
     - step: tag-analyzer-by-language

settings/import-extratags.lua

@@ -0,0 +1,129 @@
+require('flex-base')
+
+RELATION_TYPES = {
+    multipolygon = relation_as_multipolygon,
+    boundary = relation_as_multipolygon,
+    waterway = relation_as_multiline
+}
+
+MAIN_KEYS = {
+    emergency = 'always',
+    historic = 'always',
+    military = 'always',
+    natural = 'named',
+    landuse = 'named',
+    highway = {'always',
+               street_lamp = 'named',
+               traffic_signals = 'named',
+               service = 'named',
+               cycleway = 'named',
+               path = 'named',
+               footway = 'named',
+               steps = 'named',
+               bridleway = 'named',
+               track = 'named',
+               motorway_link = 'named',
+               trunk_link = 'named',
+               primary_link = 'named',
+               secondary_link = 'named',
+               tertiary_link = 'named'},
+    railway = 'named',
+    man_made = 'always',
+    aerialway = 'always',
+    boundary = {'named',
+                postal_code = 'named'},
+    aeroway = 'always',
+    amenity = 'always',
+    club = 'always',
+    craft = 'always',
+    leisure = 'always',
+    office = 'always',
+    mountain_pass = 'always',
+    shop = 'always',
+    tourism = 'always',
+    bridge = 'named_with_key',
+    tunnel = 'named_with_key',
+    waterway = 'named',
+    place = 'always'
+}
+
+MAIN_FALLBACK_KEYS = {
+    building = 'named',
+    landuse = 'named',
+    junction = 'named',
+    healthcare = 'named'
+}
+
+
+PRE_DELETE = tag_match{keys = {'note', 'note:*', 'source', 'source*', 'attribution',
+                               'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
+                               'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
+                               'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
+                               'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
+                               'ref:linz:*', 'is_in:postcode'},
+                       tags = {emergency = {'yes', 'no', 'fire_hydrant'},
+                               historic = {'yes', 'no'},
+                               military = {'yes', 'no'},
+                               natural = {'yes', 'no', 'coastline'},
+                               highway = {'no', 'turning_circle', 'mini_roundabout',
+                                          'noexit', 'crossing', 'give_way', 'stop'},
+                               railway = {'level_crossing', 'no', 'rail'},
+                               man_made = {'survey_point', 'cutline'},
+                               aerialway = {'pylon', 'no'},
+                               aeroway = {'no'},
+                               amenity = {'no'},
+                               club = {'no'},
+                               craft = {'no'},
+                               leisure = {'no'},
+                               office = {'no'},
+                               mountain_pass = {'no'},
+                               shop = {'no'},
+                               tourism = {'yes', 'no'},
+                               bridge = {'no'},
+                               tunnel = {'no'},
+                               waterway = {'riverbank'},
+                               building = {'no'},
+                               boundary = {'place'}}
+                      }
+
+POST_DELETE = tag_match{keys = {'tiger:*'}}
+
+PRE_EXTRAS = tag_match{keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
+                               'name:etymology', 'name:signed', 'name:botanical',
+                               'wikidata', '*:wikidata',
+                               'addr:street:name', 'addr:street:type'}
+                      }
+
+
+NAMES = tag_match{keys = {'name', 'name:*',
+                          'int_name', 'int_name:*',
+                          'nat_name', 'nat_name:*',
+                          'reg_name', 'reg_name:*',
+                          'loc_name', 'loc_name:*',
+                          'old_name', 'old_name:*',
+                          'alt_name', 'alt_name:*', 'alt_name_*',
+                          'official_name', 'official_name:*',
+                          'place_name', 'place_name:*',
+                          'short_name', 'short_name:*', 'brand'}}
+
+REFS = tag_match{keys = {'ref', 'int_ref', 'nat_ref', 'reg_ref', 'loc_ref', 'old_ref',
+                         'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}}
+
+POSTCODES = tag_match{keys = {'postal_code', 'postcode', 'addr:postcode',
+                              'tiger:zip_left', 'tiger:zip_right'}}
+
+COUNTRY_TAGS = tag_match{keys = {'country_code', 'ISO3166-1',
+                                 'addr:country_code', 'is_in:country_code',
+                                 'addr:country', 'is_in:country'}}
+
+HOUSENAME_TAGS = tag_match{keys = {'addr:housename'}}
+
+HOUSENUMBER_TAGS = tag_match{keys = {'addr:housenumber', 'addr:conscriptionnumber',
+                                     'addr:streetnumber'}}
+
+INTERPOLATION_TAGS = tag_match{keys = {'addr:interpolation'}}
+
+ADDRESS_TAGS = tag_match{keys = {'addr:*', 'is_in:*', 'tiger:county'}}
+
+SAVE_EXTRA_MAINS = true
+

test/bdd/api/lookup/simple.feature

@@ -31,3 +31,11 @@ Feature: Places by osm_type and osm_id Tests
       | jsonv2 |
       | geojson |
       | xml |
+
+
+    Scenario: Lookup of a linked place
+        When sending geocodejson lookup query for N1932181216
+        Then exactly 1 result is returned
+        And results contain
+          | name  |
+          | Vaduz |

test/bdd/api/search/params.feature

@@ -68,14 +68,15 @@ Feature: Search queries
           | 0 |
         Then there are duplicates
 
+    @fail-legacy
     Scenario: Search with bounded viewbox in right area
-        When sending json search query "bar" with address
+        When sending json search query "post" with address
           | bounded | viewbox |
           | 1       |  9,47,10,48 |
         Then result addresses contain
           | ID | town |
           | 0  | Vaduz |
-        When sending json search query "bar" with address
+        When sending json search query "post" with address
           | bounded | viewbox |
           | 1       |  9.49712,47.17122,9.52605,47.16242 |
         Then result addresses contain
@@ -118,18 +119,18 @@ Feature: Search queries
         Then result has centroid in 9.49712,47.16242,9.52605,47.17122
 
     Scenario: Prefer results within viewbox
-        When sending json search query "Gässle" with address
-          | accept-language |
-          | en |
-        Then result addresses contain
-          | ID | town |
-          | 0  | Balzers |
         When sending json search query "Gässle" with address
           | accept-language | viewbox |
           | en              | 9.52413,47.10759,9.53140,47.10539 |
         Then result addresses contain
           | ID | village |
           | 0  | Triesen |
+        When sending json search query "Gässle" with address
+          | accept-language | viewbox |
+          | en              | 9.45949,47.08421,9.54094,47.05466 |
+        Then result addresses contain
+          | ID | town |
+          | 0  | Balzers |
 
     Scenario: viewboxes cannot be points
         When sending json search query "foo"
@@ -171,10 +172,12 @@ Feature: Search queries
 
     Scenario: Limit number of search results
         When sending json search query "landstr"
+          | dedupe |
+          | 0      |
         Then more than 4 results are returned
         When sending json search query "landstr"
-          | limit |
+          | limit | dedupe |
-          | 4 |
+          | 4     | 0      |
         Then exactly 4 results are returned
 
     Scenario: Limit parameter must be a number

test/bdd/db/import/parenting.feature

@@ -437,6 +437,29 @@ Feature: Parenting of objects
           | object | parent_place_id |
           | N9     | R14             |
 
+
+    Scenario: Choose closest street in associatedStreet relation
+        Given the grid
+         | 1  |   |    |  | 3  |
+         | 10 |   | 11 |  | 12 |
+        And the places
+         | osm | class | type  | housenr | geometry |
+         | N1  | place | house | 1       | 1        |
+         | N3  | place | house | 3       | 3        |
+        And the named places
+         | osm  | class    | type        | geometry |
+         | W100 | highway  | residential | 10,11    |
+         | W101 | highway  | residential | 11,12    |
+        And the relations
+         | id | members                                            | tags+type |
+         | 1  | N1:house,N3:house,W100:street,W101:street | associatedStreet |
+        When importing
+        Then placex contains
+         | object | parent_place_id |
+         | N1     | W100            |
+         | N3     | W101            |
+
+
     Scenario: POIs in building inherit address
         Given the grid
          | 10 |  |   |   |   |   | 11 |

test/bdd/environment.py

@@ -27,6 +27,7 @@ userconfig = {
     'API_TEST_FILE'  : (TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf').resolve(),
     'SERVER_MODULE_PATH' : None,
     'TOKENIZER' : None, # Test with a custom tokenizer
+    'STYLE' : 'extratags',
     'PHPCOV' : False, # set to output directory to enable code coverage
 }
 

test/bdd/osm2pgsql/import/tags.feature

@@ -0,0 +1,195 @@
+@DB
+Feature: Tag evaluation
+    Tests if tags are correctly imported into the place table
+
+    Scenario: Main tags as fallback
+        When loading osm data
+            """
+            n100 Tjunction=yes,highway=bus_stop
+            n101 Tjunction=yes,name=Bar
+            n200 Tbuilding=yes,amenity=cafe
+            n201 Tbuilding=yes,name=Intersting
+            n202 Tbuilding=yes
+            """
+        Then place contains exactly
+            | object | class    | type     |
+            | N100   | highway  | bus_stop |
+            | N101   | junction | yes      |
+            | N200   | amenity  | cafe     |
+            | N201   | building | yes      |
+
+
+    Scenario: Name and reg tags
+        When loading osm data
+            """
+            n2001 Thighway=road,name=Foo,alt_name:de=Bar,ref=45
+            n2002 Thighway=road,name:prefix=Pre,name:suffix=Post,ref:de=55
+            n2003 Thighway=yes,name:%20%de=Foo,name=real1
+            n2004 Thighway=yes,name:%a%de=Foo,name=real2
+            n2005 Thighway=yes,name:%9%de=Foo,name:\\=real3
+            n2006 Thighway=yes,name:%9%de=Foo,name=rea\l3
+            """
+        Then place contains exactly
+            | object | class   | type | name |
+            | N2001  | highway | road | 'name': 'Foo', 'alt_name:de': 'Bar', 'ref': '45' |
+            | N2002  | highway | road | - |
+            | N2003  | highway | yes  | 'name: de': 'Foo', 'name': 'real1' |
+            | N2004  | highway | yes  | 'name:\nde': 'Foo', 'name': 'real2' |
+            | N2005  | highway | yes  | 'name:\tde': 'Foo', 'name:\\\\': 'real3' |
+            | N2006  | highway | yes  | 'name:\tde': 'Foo', 'name': 'rea\\l3' |
+
+        And place contains
+            | object | extratags |
+            | N2002  | 'name:prefix': 'Pre', 'name:suffix': 'Post', 'ref:de': '55' |
+
+
+    Scenario: Name when using with_name flag
+        When loading osm data
+            """
+            n3001 Tbridge=yes,bridge:name=GoldenGate
+            n3002 Tbridge=yes,bridge:name:en=Rainbow
+            """
+        Then place contains exactly
+            | object | class   | type | name                 |
+            | N3001  | bridge  | yes  | 'name': 'GoldenGate' |
+            | N3002  | bridge  | yes  | 'name:en': 'Rainbow' |
+
+
+    Scenario: Address tags
+        When loading osm data
+            """
+            n4001 Taddr:housenumber=34,addr:city=Esmarald,addr:county=Land
+            n4002 Taddr:streetnumber=10,is_in:city=Rootoo,is_in=Gold
+            """
+        Then place contains exactly
+            | object | class | address             |
+            | N4001  | place | 'housenumber': '34', 'city': 'Esmarald', 'county': 'Land' |
+            | N4002  | place | 'streetnumber': '10', 'city': 'Rootoo' |
+
+
+    Scenario: Country codes
+        When loading osm data
+            """
+            n5001 Tshop=yes,country_code=DE
+            n5002 Tshop=yes,country_code=toolong
+            n5003 Tshop=yes,country_code=x
+            n5004 Tshop=yes,addr:country=us
+            n5005 Tshop=yes,country=be
+            n5006 Tshop=yes,addr:country=France
+            """
+        Then place contains exactly
+            | object | class | address         |
+            | N5001  | shop  | 'country': 'DE' |
+            | N5002  | shop  | - |
+            | N5003  | shop  | - |
+            | N5004  | shop  | 'country': 'us' |
+            | N5005  | shop  | - |
+            | N5006  | shop  | - |
+
+
+    Scenario: Postcodes
+        When loading osm data
+            """
+            n6001 Tshop=bank,addr:postcode=12345
+            n6002 Tshop=bank,tiger:zip_left=34343
+            n6003 Tshop=bank,is_in:postcode=9009
+            """
+        Then place contains exactly
+            | object | class | address             |
+            | N6001  | shop  | 'postcode': '12345' |
+            | N6002  | shop  | 'postcode': '34343' |
+            | N6003  | shop  | -                   |
+
+
+    Scenario: Main with extra
+        When loading osm data
+            """
+            n7001 Thighway=primary,bridge=yes,name=1
+            n7002 Thighway=primary,bridge=yes,bridge:name=1
+            """
+        Then place contains exactly
+            | object        | class   | type    | name        | extratags+bridge:name |
+            | N7001         | highway | primary | 'name': '1' | -                     |
+            | N7002:highway | highway | primary | -           | 1                     |
+            | N7002:bridge  | bridge  | yes     | 'name': '1' | 1                     |
+
+
+    Scenario: Global fallback and skipping
+        When loading osm data
+            """
+            n8001 Tshop=shoes,note:de=Nein,xx=yy
+            n8002 Tshop=shoes,building=no,ele=234
+            n8003 Tshop=shoes,name:source=survey
+            """
+        Then place contains exactly
+            | object | class | extratags    |
+            | N8001  | shop  | 'xx': 'yy'   |
+            | N8002  | shop  | 'ele': '234' |
+            | N8003  | shop  | -            |
+
+
+    Scenario: Admin levels
+        When loading osm data
+            """
+            n9001 Tplace=city
+            n9002 Tplace=city,admin_level=16
+            n9003 Tplace=city,admin_level=x
+            n9004 Tplace=city,admin_level=1
+            n9005 Tplace=city,admin_level=0
+            n9006 Tplace=city,admin_level=2.5
+            """
+        Then place contains exactly
+            | object | class | admin_level |
+            | N9001  | place | 15          |
+            | N9002  | place | 15          |
+            | N9003  | place | 15          |
+            | N9004  | place | 1           |
+            | N9005  | place | 15          |
+            | N9006  | place | 15          |
+
+
+    Scenario: Administrative boundaries with place tags
+        When loading osm data
+            """
+            n10001 Tboundary=administrative,place=city,name=A
+            n10002 Tboundary=natural,place=city,name=B
+            n10003 Tboundary=administrative,place=island,name=C
+            """
+        Then place contains
+            | object          | class    | type           | extratags       |
+            | N10001          | boundary | administrative | 'place': 'city' |
+        And place contains
+            | object          | class    | type           |
+            | N10002:boundary | boundary | natural        |
+            | N10002:place    | place    | city           |
+            | N10003:boundary | boundary | administrative |
+            | N10003:place    | place    | island         |
+
+
+    Scenario: Building fallbacks
+        When loading osm data
+            """
+            n12001 Ttourism=hotel,building=yes
+            n12002 Tbuilding=house
+            n12003 Tbuilding=shed,addr:housenumber=1
+            n12004 Tbuilding=yes,name=Das-Haus
+            n12005 Tbuilding=yes,addr:postcode=12345
+            """
+        Then place contains exactly
+            | object | class    | type     |
+            | N12001 | tourism  | hotel    |
+            | N12003 | building | shed     |
+            | N12004 | building | yes      |
+            | N12005 | place    | postcode |
+
+
+    Scenario: Address interpolations
+        When loading osm data
+            """
+            n13001 Taddr:interpolation=odd
+            n13002 Taddr:interpolation=even,place=city
+            """
+        Then place contains exactly
+            | object | class | type   | address                 |
+            | N13001 | place | houses | 'interpolation': 'odd'  |
+            | N13002 | place | houses | 'interpolation': 'even' |

test/bdd/osm2pgsql/update/interpolations.feature

@@ -0,0 +1,133 @@
+@DB
+Feature: Updates of address interpolation objects
+    Test that changes to address interpolation objects are correctly
+    propagated.
+
+    Background:
+        Given the grid
+            | 1 | 2 |
+
+
+    Scenario: Adding a new interpolation
+        When loading osm data
+            """
+            n1 Taddr:housenumber=3
+            n2 Taddr:housenumber=17
+            """
+        Then place contains
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+
+        When updating osm data
+            """
+            w99 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+            | W99:place | houses |
+        When indexing
+        Then placex contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+        Then location_property_osmline contains exactly
+            | object |
+            | 99:5   |
+
+
+    Scenario: Delete an existing interpolation
+        When loading osm data
+            """
+            n1 Taddr:housenumber=2
+            n2 Taddr:housenumber=7
+            w99 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+            | W99:place | houses |
+
+        When updating osm data
+            """
+            w99 v2 dD
+            """
+        Then place contains
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+        When indexing
+        Then placex contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+        Then location_property_osmline contains exactly
+            | object | indexed_status |
+
+
+    Scenario: Changing an object to an interpolation
+        When loading osm data
+            """
+            n1 Taddr:housenumber=3
+            n2 Taddr:housenumber=17
+            w99 Thighway=residential Nn1,n2
+            """
+        Then place contains
+            | object      | type   |
+            | N1:place    | house  |
+            | N2:place    | house  |
+            | W99:highway | residential  |
+
+        When updating osm data
+            """
+            w99 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+            | W99:place | houses |
+        When indexing
+        Then placex contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+        And location_property_osmline contains exactly
+            | object |
+            | 99:5   |
+
+
+    Scenario: Changing an interpolation to something else
+        When loading osm data
+            """
+            n1 Taddr:housenumber=3
+            n2 Taddr:housenumber=17
+            w99 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains
+            | object      | type   |
+            | N1:place    | house  |
+            | N2:place    | house  |
+            | W99:place | houses |
+
+        When updating osm data
+            """
+            w99 Thighway=residential Nn1,n2
+            """
+        Then place contains
+            | object      | type   |
+            | N1:place    | house  |
+            | N2:place    | house  |
+            | W99:highway | residential  |
+        When indexing
+        Then placex contains exactly
+            | object      | type   |
+            | N1:place    | house  |
+            | N2:place    | house  |
+            | W99:highway | residential  |
+        And location_property_osmline contains exactly
+            | object |
+

test/bdd/osm2pgsql/update/postcodes.feature

@@ -0,0 +1,163 @@
+@DB
+Feature: Update of postcode only objects
+    Tests that changes to objects containing only a postcode are
+    propagated correctly.
+
+
+    Scenario: Adding a postcode-only node
+        When loading osm data
+            """
+            """
+        Then place contains exactly
+            | object |
+
+        When updating osm data
+            """
+            n34 Tpostcode=4456
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N34:place | postcode |
+        When indexing
+        Then placex contains exactly
+            | object |
+
+
+    Scenario: Deleting a postcode-only node
+        When loading osm data
+            """
+            n34 Tpostcode=4456
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N34:place | postcode |
+
+        When updating osm data
+            """
+            n34 v2 dD
+            """
+        Then place contains exactly
+            | object |
+        When indexing
+        Then placex contains exactly
+            | object |
+
+
+    Scenario Outline: Converting a regular object into a postcode-only node
+        When loading osm data
+            """
+            n34 T<class>=<type>
+            """
+        Then place contains exactly
+            | object      | type   |
+            | N34:<class> | <type> |
+
+        When updating osm data
+            """
+            n34 Tpostcode=4456
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N34:place | postcode |
+        When indexing
+        Then placex contains exactly
+            | object |
+
+        Examples:
+            | class   | type       |
+            | amenity | restaurant |
+            | place   | hamlet     |
+
+
+    Scenario Outline: Converting a postcode-only node into a regular object
+        When loading osm data
+            """
+            n34 Tpostcode=4456
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N34:place | postcode |
+
+        When updating osm data
+            """
+            n34 T<class>=<type>
+            """
+        Then place contains exactly
+            | object      | type   |
+            | N34:<class> | <type> |
+        When indexing
+        Then placex contains exactly
+            | object      | type   |
+            | N34:<class> | <type> |
+
+        Examples:
+            | class   | type       |
+            | amenity | restaurant |
+            | place   | hamlet     |
+
+
+    Scenario: Converting na interpolation into a postcode-only node
+        Given the grid
+            | 1 | 2 |
+        When loading osm data
+            """
+            n1 Taddr:housenumber=3
+            n2 Taddr:housenumber=17
+            w34 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+            | W34:place | houses |
+
+        When updating osm data
+            """
+            w34 Tpostcode=4456 Nn1,n2
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N1:place  | house    |
+            | N2:place  | house    |
+            | W34:place | postcode |
+        When indexing
+        Then location_property_osmline contains exactly
+            | object |
+        And placex contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+
+
+    Scenario: Converting a postcode-only node into an interpolation
+        Given the grid
+            | 1 | 2 |
+        When loading osm data
+            """
+            n1 Taddr:housenumber=3
+            n2 Taddr:housenumber=17
+            w34 Tpostcode=4456 Nn1,n2
+            """
+        Then place contains exactly
+            | object    | type     |
+            | N1:place  | house    |
+            | N2:place  | house    |
+            | W34:place | postcode |
+
+        When updating osm data
+            """
+            w34 Taddr:interpolation=odd Nn1,n2
+            """
+        Then place contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |
+            | W34:place | houses |
+        When indexing
+        Then location_property_osmline contains exactly
+            | object |
+            | 34:5   |
+        And placex contains exactly
+            | object    | type   |
+            | N1:place  | house  |
+            | N2:place  | house  |

test/bdd/osm2pgsql/update/simple.feature

@@ -2,60 +2,48 @@
 Feature: Update of simple objects by osm2pgsql
     Testing basic update functions of osm2pgsql.
 
-    Scenario: Import object with two main tags
+    Scenario: Adding a new object
         When loading osm data
           """
-          n1 Ttourism=hotel,amenity=restaurant,name=foo
+          n1 Tplace=town,name=Middletown
-          n2 Tplace=locality,name=spotty
           """
-        Then place contains
+        Then place contains exactly
-          | object     | type       | name+name |
+          | object   | type | name+name  |
-          | N1:tourism | hotel      | foo |
+          | N1:place | town | Middletown |
-          | N1:amenity | restaurant | foo |
-          | N2:place   | locality   | spotty |
-        When updating osm data
-          """
-          n1 dV Ttourism=hotel,name=foo
-          n2 dD
-          """
-        Then place has no entry for N1:amenity
-        And place has no entry for N2
-        And place contains
-          | object     | class   | type       | name |
-          | N1:tourism | tourism | hotel      | 'name' : 'foo' |
 
-    Scenario: Downgrading a highway to one that is dropped without name
+       When updating osm data
-        When loading osm data
+         """
-          """
+         n2 Tamenity=hotel,name=Posthotel
-          n100 x0 y0
+         """
-          n101 x0.0001 y0.0001
+        Then place contains exactly
-          w1 Thighway=residential Nn100,n101
+          | object     | type  | name+name  |
-          """
+          | N1:place   | town  | Middletown |
-        Then place contains
+          | N2:amenity | hotel | Posthotel  |
-          | object     |
+        And placex contains exactly
-          | W1:highway |
+          | object     | type  | name+name  | indexed_status |
-        When updating osm data
+          | N1:place   | town  | Middletown | 0              |
-          """
+          | N2:amenity | hotel | Posthotel  | 1              |
-          w1 Thighway=service Nn100,n101
-          """
-        Then place has no entry for W1
 
-    Scenario: Downgrading a highway when a second tag is present
+
+    Scenario: Deleting an existing object
         When loading osm data
           """
-          n100 x0 y0
+          n1 Tplace=town,name=Middletown
-          n101 x0.0001 y0.0001
+          n2 Tamenity=hotel,name=Posthotel
-          w1 Thighway=residential,tourism=hotel Nn100,n101
           """
-        Then place contains
+        Then place contains exactly
-          | object     |
+          | object     | type  | name+name  |
-          | W1:highway |
+          | N1:place   | town  | Middletown |
-          | W1:tourism |
+          | N2:amenity | hotel | Posthotel  |
-        When updating osm data
+
-          """
+       When updating osm data
-          w1 Thighway=service,tourism=hotel Nn100,n101
+         """
-          """
+         n2 dD
-        Then place has no entry for W1:highway
+         """
-        And place contains
+        Then place contains exactly
-          | object     |
+          | object     | type  | name+name  |
-          | W1:tourism |
+          | N1:place   | town  | Middletown |
+        And placex contains exactly
+          | object     | type  | name+name  | indexed_status |
+          | N1:place   | town  | Middletown | 0              |
+          | N2:amenity | hotel | Posthotel  | 100            |

test/bdd/osm2pgsql/update/tags.feature

@@ -0,0 +1,490 @@
+@DB
+Feature: Tag evaluation
+    Tests if tags are correctly updated in the place table
+
+    Background:
+        Given the grid
+            | 1  | 2  | 3 |
+            | 10 | 11 |   |
+            | 45 | 46 |   |
+
+    Scenario: Main tag deleted
+        When loading osm data
+            """
+            n1 Tamenity=restaurant
+            n2 Thighway=bus_stop,railway=stop,name=X
+            n3 Tamenity=prison
+            """
+        Then place contains exactly
+            | object     | class   | type       |
+            | N1         | amenity | restaurant |
+            | N2:highway | highway | bus_stop   |
+            | N2:railway | railway | stop       |
+            | N3         | amenity | prison     |
+
+        When updating osm data
+            """
+            n1 Tnot_a=restaurant
+            n2 Thighway=bus_stop,name=X
+            """
+        Then place contains exactly
+            | object     | class   | type       |
+            | N2:highway | highway | bus_stop   |
+            | N3         | amenity | prison     |
+        And placex contains
+            | object     | indexed_status |
+            | N3:amenity | 0              |
+        When indexing
+        Then placex contains exactly
+            | object     | type     | name        |
+            | N2:highway | bus_stop | 'name': 'X' |
+            | N3:amenity | prison   | -           |
+
+
+    Scenario: Main tag added
+        When loading osm data
+            """
+            n1 Tatity=restaurant
+            n2 Thighway=bus_stop,name=X
+            """
+        Then place contains exactly
+            | object     | class   | type       |
+            | N2:highway | highway | bus_stop   |
+
+        When updating osm data
+            """
+            n1 Tamenity=restaurant
+            n2 Thighway=bus_stop,railway=stop,name=X
+            """
+        Then place contains exactly
+            | object     | class   | type       |
+            | N1         | amenity | restaurant |
+            | N2:highway | highway | bus_stop   |
+            | N2:railway | railway | stop       |
+        When indexing
+        Then placex contains exactly
+            | object     | type       | name        |
+            | N1:amenity | restaurant | -           |
+            | N2:highway | bus_stop   | 'name': 'X' |
+            | N2:railway | stop       | 'name': 'X' |
+
+
+    Scenario: Main tag modified
+        When loading osm data
+            """
+            n10 Thighway=footway,name=X
+            n11 Tamenity=atm
+            """
+        Then place contains exactly
+            | object | class   | type    |
+            | N10    | highway | footway |
+            | N11    | amenity | atm     |
+
+        When updating osm data
+            """
+            n10 Thighway=path,name=X
+            n11 Thighway=primary
+            """
+        Then place contains exactly
+            | object | class   | type    |
+            | N10    | highway | path    |
+            | N11    | highway | primary |
+        When indexing
+        Then placex contains exactly
+            | object      | type       | name        |
+            | N10:highway | path       | 'name': 'X' |
+            | N11:highway | primary    | -           |
+
+
+    Scenario: Main tags with name, name added
+        When loading osm data
+            """
+            n45 Tlanduse=cemetry
+            n46 Tbuilding=yes
+            """
+        Then place contains exactly
+            | object | class   | type    |
+
+        When updating osm data
+            """
+            n45 Tlanduse=cemetry,name=TODO
+            n46 Tbuilding=yes,addr:housenumber=1
+            """
+        Then place contains exactly
+            | object | class   | type    |
+            | N45    | landuse | cemetry |
+            | N46    | building| yes     |
+        When indexing
+        Then placex contains exactly
+            | object      | type       | name           | address            |
+            | N45:landuse | cemetry    | 'name': 'TODO' | -                  |
+            | N46:building| yes        | -              | 'housenumber': '1' |
+
+
+    Scenario: Main tags with name, name removed
+        When loading osm data
+            """
+            n45 Tlanduse=cemetry,name=TODO
+            n46 Tbuilding=yes,addr:housenumber=1
+            """
+        Then place contains exactly
+            | object | class   | type    |
+            | N45    | landuse | cemetry |
+            | N46    | building| yes     |
+
+        When updating osm data
+            """
+            n45 Tlanduse=cemetry
+            n46 Tbuilding=yes
+            """
+        Then place contains exactly
+            | object | class   | type    |
+        When indexing
+        Then placex contains exactly
+            | object      |
+
+    Scenario: Main tags with name, name modified
+        When loading osm data
+            """
+            n45 Tlanduse=cemetry,name=TODO
+            n46 Tbuilding=yes,addr:housenumber=1
+            """
+        Then place contains exactly
+            | object | class   | type    | name            | address           |
+            | N45    | landuse | cemetry | 'name' : 'TODO' | -                 |
+            | N46    | building| yes     | -               | 'housenumber': '1'|
+
+        When updating osm data
+            """
+            n45 Tlanduse=cemetry,name=DONE
+            n46 Tbuilding=yes,addr:housenumber=10
+            """
+        Then place contains exactly
+            | object | class   | type    | name            | address            |
+            | N45    | landuse | cemetry | 'name' : 'DONE' | -                  |
+            | N46    | building| yes     | -               | 'housenumber': '10'|
+        When indexing
+        Then placex contains exactly
+            | object | class   | type    | name            | address            |
+            | N45    | landuse | cemetry | 'name' : 'DONE' | -                  |
+            | N46    | building| yes     | -               | 'housenumber': '10'|
+
+
+    Scenario: Main tag added to address only node
+        When loading osm data
+            """
+            n1 Taddr:housenumber=345
+            """
+        Then place contains exactly
+            | object | class | type  | address |
+            | N1     | place | house | 'housenumber': '345'|
+
+        When updating osm data
+            """
+            n1 Taddr:housenumber=345,building=yes
+            """
+        Then place contains exactly
+            | object | class    | type  | address |
+            | N1     | building | yes   | 'housenumber': '345'|
+        When indexing
+        Then placex contains exactly
+            | object | class    | type  | address |
+            | N1     | building | yes   | 'housenumber': '345'|
+
+
+    Scenario: Main tag removed from address only node
+        When loading osm data
+            """
+            n1 Taddr:housenumber=345,building=yes
+            """
+        Then place contains exactly
+            | object | class    | type  | address |
+            | N1     | building | yes   | 'housenumber': '345'|
+
+        When updating osm data
+            """
+            n1 Taddr:housenumber=345
+            """
+        Then place contains exactly
+            | object | class | type  | address |
+            | N1     | place | house | 'housenumber': '345'|
+        When indexing
+        Then placex contains exactly
+            | object | class | type  | address |
+            | N1     | place | house | 'housenumber': '345'|
+
+
+    Scenario: Main tags with name key, adding key name
+        When loading osm data
+            """
+            n2 Tbridge=yes
+            """
+        Then place contains exactly
+            | object | class    | type  |
+
+        When updating osm data
+            """
+            n2 Tbridge=yes,bridge:name=high
+            """
+        Then place contains exactly
+            | object | class    | type  | name           |
+            | N2     | bridge   | yes   | 'name': 'high' |
+        When indexing
+        Then placex contains exactly
+            | object | class    | type  | name           |
+            | N2     | bridge   | yes   | 'name': 'high' |
+
+
+    Scenario: Main tags with name key, deleting key name
+        When loading osm data
+            """
+            n2 Tbridge=yes,bridge:name=high
+            """
+        Then place contains exactly
+            | object | class    | type  | name           |
+            | N2     | bridge   | yes   | 'name': 'high' |
+
+        When updating osm data
+            """
+            n2 Tbridge=yes
+            """
+        Then place contains exactly
+            | object |
+        When indexing
+        Then placex contains exactly
+            | object |
+
+
+    Scenario: Main tags with name key, changing key name
+        When loading osm data
+            """
+            n2 Tbridge=yes,bridge:name=high
+            """
+        Then place contains exactly
+            | object | class    | type  | name           |
+            | N2     | bridge   | yes   | 'name': 'high' |
+
+        When updating osm data
+            """
+            n2 Tbridge=yes,bridge:name:en=high
+            """
+        Then place contains exactly
+            | object | class  | type | name              |
+            | N2     | bridge | yes  | 'name:en': 'high' |
+        When indexing
+        Then placex contains exactly
+            | object | class  | type | name              |
+            | N2     | bridge | yes  | 'name:en': 'high' |
+
+
+    Scenario: Downgrading a highway to one that is dropped without name
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=residential Nn100,n101
+          """
+        Then place contains exactly
+          | object     |
+          | W1:highway |
+
+        When updating osm data
+          """
+          w1 Thighway=service Nn100,n101
+          """
+        Then place contains exactly
+          | object     |
+        When indexing
+        Then placex contains exactly
+            | object |
+
+
+    Scenario: Upgrading a highway to one that is not dropped without name
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=service Nn100,n101
+          """
+        Then place contains exactly
+          | object     |
+
+        When updating osm data
+          """
+          w1 Thighway=unclassified Nn100,n101
+          """
+        Then place contains exactly
+          | object     |
+          | W1:highway |
+        When indexing
+        Then placex contains exactly
+          | object     |
+          | W1:highway |
+
+
+    Scenario: Downgrading a highway when a second tag is present
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=residential,tourism=hotel Nn100,n101
+          """
+        Then place contains exactly
+          | object     | type        |
+          | W1:highway | residential |
+          | W1:tourism | hotel       |
+
+        When updating osm data
+          """
+          w1 Thighway=service,tourism=hotel Nn100,n101
+          """
+        Then place contains exactly
+          | object     | type  |
+          | W1:tourism | hotel |
+        When indexing
+        Then placex contains exactly
+          | object     | type  |
+          | W1:tourism | hotel |
+
+
+    Scenario: Upgrading a highway when a second tag is present
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=service,tourism=hotel Nn100,n101
+          """
+        Then place contains exactly
+          | object     | type  |
+          | W1:tourism | hotel |
+
+        When updating osm data
+          """
+          w1 Thighway=residential,tourism=hotel Nn100,n101
+          """
+        Then place contains exactly
+          | object     | type        |
+          | W1:highway | residential |
+          | W1:tourism | hotel       |
+        When indexing
+        Then placex contains exactly
+          | object     | type        |
+          | W1:highway | residential |
+          | W1:tourism | hotel       |
+
+
+    Scenario: Replay on administrative boundary
+        When loading osm data
+          """
+          n10 x34.0 y-4.23
+          n11 x34.1 y-4.23
+          n12 x34.2 y-4.13
+          w10 Tboundary=administrative,waterway=river,name=Border,admin_level=2 Nn12,n11,n10
+          """
+        Then place contains exactly
+          | object       | type           | admin_level | name             |
+          | W10:waterway | river          | 2           | 'name': 'Border' |
+          | W10:boundary | administrative | 2           | 'name': 'Border' |
+
+        When updating osm data
+          """
+          w10 Tboundary=administrative,waterway=river,name=Border,admin_level=2 Nn12,n11,n10
+          """
+        Then place contains exactly
+          | object       | type           | admin_level | name             |
+          | W10:waterway | river          | 2           | 'name': 'Border' |
+          | W10:boundary | administrative | 2           | 'name': 'Border' |
+        When indexing
+        Then placex contains exactly
+          | object       | type           | admin_level | name             |
+          | W10:waterway | river          | 2           | 'name': 'Border' |
+
+
+    Scenario: Change admin_level on administrative boundary
+        Given the grid
+          | 10 | 11 |
+          | 13 | 12 |
+        When loading osm data
+          """
+          n10
+          n11
+          n12
+          n13
+          w10 Nn10,n11,n12,n13,n10
+          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=2 Mw10@
+          """
+        Then place contains exactly
+          | object       | admin_level |
+          | R10:boundary | 2           |
+
+        When updating osm data
+          """
+          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
+          """
+        Then place contains exactly
+          | object       | type           | admin_level |
+          | R10:boundary | administrative | 4           |
+        When indexing
+        Then placex contains exactly
+          | object       | type           | admin_level |
+          | R10:boundary | administrative | 4           |
+
+
+    Scenario: Change boundary to administrative
+        Given the grid
+          | 10 | 11 |
+          | 13 | 12 |
+        When loading osm data
+          """
+          n10
+          n11
+          n12
+          n13
+          w10 Nn10,n11,n12,n13,n10
+          r10 Ttype=multipolygon,boundary=informal,name=Border,admin_level=4 Mw10@
+          """
+        Then place contains exactly
+          | object       | type     | admin_level |
+          | R10:boundary | informal | 4           |
+
+        When updating osm data
+          """
+          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
+          """
+        Then place contains exactly
+          | object       | type           | admin_level |
+          | R10:boundary | administrative | 4           |
+        When indexing
+        Then placex contains exactly
+          | object       | type           | admin_level |
+          | R10:boundary | administrative | 4           |
+
+
+    Scenario: Change boundary away from administrative
+        Given the grid
+          | 10 | 11 |
+          | 13 | 12 |
+        When loading osm data
+          """
+          n10
+          n11
+          n12
+          n13
+          w10 Nn10,n11,n12,n13,n10
+          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
+          """
+        Then place contains exactly
+          | object       | type           | admin_level |
+          | R10:boundary | administrative | 4           |
+
+        When updating osm data
+          """
+          r10 Ttype=multipolygon,boundary=informal,name=Border,admin_level=4 Mw10@
+          """
+        Then place contains exactly
+          | object       | type     | admin_level |
+          | R10:boundary | informal | 4           |
+        When indexing
+        Then placex contains exactly
+          | object       | type     | admin_level |
+          | R10:boundary | informal | 4           |

test/bdd/steps/nominatim_environment.py

@@ -36,6 +36,7 @@ class NominatimEnvironment:
         self.api_test_db = config['API_TEST_DB']
         self.api_test_file = config['API_TEST_FILE']
         self.tokenizer = config['TOKENIZER']
+        self.import_style = config['STYLE']
         self.server_module_path = config['SERVER_MODULE_PATH']
         self.reuse_template = not config['REMOVE_TEMPLATE']
         self.keep_scenario_db = config['KEEP_TEST_DB']
@@ -107,6 +108,8 @@ class NominatimEnvironment:
         self.test_env['NOMINATIM_NOMINATIM_TOOL'] = str((self.build_dir / 'nominatim').resolve())
         if self.tokenizer is not None:
             self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer
+        if self.import_style is not None:
+            self.test_env['NOMINATIM_IMPORT_STYLE'] = self.import_style
 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
@@ -201,19 +204,21 @@ class NominatimEnvironment:
             self.api_db_done = True
 
             if not self._reuse_or_drop_db(self.api_test_db):
-                testdata = Path('__file__') / '..' / '..' / 'testdb'
+                testdata = (Path(__file__) / '..' / '..' / '..' / 'testdb').resolve()
-                self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
+                self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata)
+                simp_file = Path(self.website_dir.name) / 'secondary_importance.sql.gz'
+                simp_file.symlink_to(testdata / 'secondary_importance.sql.gz')
 
                 try:
                     self.run_nominatim('import', '--osm-file', str(self.api_test_file))
-                    self.run_nominatim('add-data', '--tiger-data', str((testdata / 'tiger').resolve()))
+                    self.run_nominatim('add-data', '--tiger-data', str(testdata / 'tiger'))
                     self.run_nominatim('freeze')
 
                     if self.tokenizer == 'legacy':
-                        phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
+                        phrase_file = str(testdata / 'specialphrases_testdb.sql')
                         run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
                     else:
-                        csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
+                        csv_path = str(testdata / 'full_en_phrases_test.csv')
                         self.run_nominatim('special-phrases', '--import-from-csv', csv_path)
                 except:
                     self.db_drop_database(self.api_test_db)

test/bdd/steps/place_inserter.py

@@ -92,6 +92,12 @@ class PlaceColumn:
         else:
             self.columns[column] = {key: value}
 
+    def db_delete(self, cursor):
+        """ Issue a delete for the given OSM object.
+        """
+        cursor.execute('DELETE FROM place WHERE osm_type = %s and osm_id = %s',
+                       (self.columns['osm_type'] , self.columns['osm_id']))
+
     def db_insert(self, cursor):
         """ Insert the collected data into the database.
         """

test/bdd/steps/steps_db_ops.py

@@ -118,7 +118,10 @@ def update_place_table(context):
     context.nominatim.run_nominatim('refresh', '--functions')
     with context.db.cursor() as cur:
         for row in context.table:
-            PlaceColumn(context).add_row(row, False).db_insert(cur)
+            col = PlaceColumn(context).add_row(row, False)
+            col.db_delete(cur)
+            col.db_insert(cur)
+        cur.execute('SELECT flush_deleted_places()')
 
     context.nominatim.reindex_placex(context.db)
     check_database_integrity(context)
@@ -143,8 +146,10 @@ def delete_places(context, oids):
     """
     context.nominatim.run_nominatim('refresh', '--functions')
     with context.db.cursor() as cur:
+        cur.execute('TRUNCATE place_to_be_deleted')
         for oid in oids.split(','):
             NominatimID(oid).query_osm_id(cur, 'DELETE FROM place WHERE {}')
+        cur.execute('SELECT flush_deleted_places()')
 
     context.nominatim.reindex_placex(context.db)
 
@@ -185,7 +190,10 @@ def check_place_contents(context, table, exact):
 
         if exact:
             cur.execute('SELECT osm_type, osm_id, class from {}'.format(table))
-            assert expected_content == set([(r[0], r[1], r[2]) for r in cur])
+            actual = set([(r[0], r[1], r[2]) for r in cur])
+            assert expected_content == actual, \
+                   f"Missing entries: {expected_content - actual}\n" \
+                   f"Not expected in table: {actual - expected_content}"
 
 
 @then("(?P<table>placex|place) has no entry for (?P<oid>.*)")
@@ -372,4 +380,49 @@ def check_location_property_osmline(context, oid, neg):
 
         assert not todo, f"Unmatched lines in table: {list(context.table[i] for i in todo)}"
 
+@then("location_property_osmline contains(?P<exact> exactly)?")
+def check_place_contents(context, exact):
+    """ Check contents of the interpolation table. Each row represents a table row
+        and all data must match. Data not present in the expected table, may
+        be arbitry. The rows are identified via the 'object' column which must
+        have an identifier of the form '<osm id>[:<startnumber>]'. When multiple
+        rows match (for example because 'startnumber' was left out and there are
+        multiple entries for the given OSM object) then all must match. All
+        expected rows are expected to be present with at least one database row.
+        When 'exactly' is given, there must not be additional rows in the database.
+    """
+    with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
+        expected_content = set()
+        for row in context.table:
+            if ':' in row['object']:
+                nid, start = row['object'].split(':', 2)
+                start = int(start)
+            else:
+                nid, start = row['object'], None
+
+            query = """SELECT *, ST_AsText(linegeo) as geomtxt,
+                              ST_GeometryType(linegeo) as geometrytype
+                       FROM location_property_osmline WHERE osm_id=%s"""
+
+            if ':' in row['object']:
+                query += ' and startnumber = %s'
+                params = [int(val) for val in row['object'].split(':', 2)]
+            else:
+                params = (int(row['object']), )
+
+            cur.execute(query, params)
+            assert cur.rowcount > 0, "No rows found for " + row['object']
+
+            for res in cur:
+                if exact:
+                    expected_content.add((res['osm_id'], res['startnumber']))
+
+                DBRow(nid, res, context).assert_row(row, ['object'])
+
+        if exact:
+            cur.execute('SELECT osm_id, startnumber from location_property_osmline')
+            actual = set([(r[0], r[1]) for r in cur])
+            assert expected_content == actual, \
+                   f"Missing entries: {expected_content - actual}\n" \
+                   f"Not expected in table: {actual - expected_content}"
 

test/bdd/steps/steps_osm_data.py

@@ -10,6 +10,7 @@ import os
 from pathlib import Path
 
 from nominatim.tools.exec_utils import run_osm2pgsql
+from nominatim.tools.replication import run_osm2pgsql_updates
 
 from geometry_alias import ALIASES
 
@@ -17,7 +18,8 @@ def get_osm2pgsql_options(nominatim_env, fname, append):
     return dict(import_file=fname,
                 osm2pgsql=str(nominatim_env.build_dir / 'osm2pgsql' / 'osm2pgsql'),
                 osm2pgsql_cache=50,
-                osm2pgsql_style=str(nominatim_env.src_dir / 'settings' / 'import-extratags.style'),
+                osm2pgsql_style=str(nominatim_env.get_test_config().get_import_style_file()),
+                osm2pgsql_style_path=nominatim_env.get_test_config().config_dir,
                 threads=1,
                 dsn=nominatim_env.get_libpq_dsn(),
                 flatnode_file='',
@@ -117,6 +119,15 @@ def update_from_osm_file(context):
     # create an OSM file and import it
     fname = write_opl_file(context.text, context.osm)
     try:
-        run_osm2pgsql(get_osm2pgsql_options(context.nominatim, fname, append=True))
+        run_osm2pgsql_updates(context.db,
+                              get_osm2pgsql_options(context.nominatim, fname, append=True))
     finally:
         os.remove(fname)
+
+@when('indexing')
+def index_database(context):
+    """
+    Run the Nominatim indexing step. This will process data previously
+    loaded with 'updating osm data'
+    """
+    context.nominatim.run_nominatim('index')

test/python/cli/test_cmd_import.py

@@ -40,6 +40,7 @@ class TestCliImportWithDb:
             mock_func_factory(nominatim.data.country_info, 'setup_country_tables'),
             mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
             mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
+            mock_func_factory(nominatim.tools.refresh, 'import_secondary_importance'),
             mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
             mock_func_factory(nominatim.tools.database_import, 'load_data'),
             mock_func_factory(nominatim.tools.database_import, 'create_tables'),
@@ -105,11 +106,8 @@ class TestCliImportWithDb:
         for mock in mocks:
             assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
 
-        assert temp_db_conn.index_exists('idx_placex_pendingsector')
-
         # Calling it again still works for the index
         assert self.call_nominatim('import', '--continue', 'indexing') == 0
-        assert temp_db_conn.index_exists('idx_placex_pendingsector')
 
 
     def test_import_continue_postprocess(self, mock_func_factory):

test/python/cli/test_cmd_refresh.py

@@ -71,6 +71,18 @@ class TestRefresh:
 
         assert self.call_nominatim('refresh', '--wiki-data') == 1
 
+    def test_refresh_secondary_importance_file_not_found(self):
+        assert self.call_nominatim('refresh', '--secondary-importance') == 1
+
+
+    def test_refresh_secondary_importance_new_table(self, mock_func_factory):
+        mocks = [mock_func_factory(nominatim.tools.refresh, 'import_secondary_importance'),
+                 mock_func_factory(nominatim.tools.refresh, 'create_functions')]
+
+        assert self.call_nominatim('refresh', '--secondary-importance') == 0
+        assert mocks[0].called == 1
+        assert mocks[1].called == 1
+
 
     def test_refresh_importance_computed_after_wiki_import(self, monkeypatch):
         calls = []

test/python/tokenizer/sanitizers/test_clean_tiger_tags.py

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2022 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Tests for sanitizer that clean up TIGER tags.
+"""
+import pytest
+
+from nominatim.tokenizer.place_sanitizer import PlaceSanitizer
+from nominatim.data.place_info import PlaceInfo
+
+class TestCleanTigerTags:
+
+    @pytest.fixture(autouse=True)
+    def setup_country(self, def_config):
+        self.config = def_config
+
+
+    def run_sanitizer_on(self, addr):
+        place = PlaceInfo({'address': addr})
+        _, outaddr = PlaceSanitizer([{'step': 'clean-tiger-tags'}], self.config).process_names(place)
+
+        return sorted([(p.name, p.kind, p.suffix) for p in outaddr])
+
+    @pytest.mark.parametrize('inname,outname', [('Hamilton, AL', 'Hamilton'),
+                                                ('Little, Borough, CA', 'Little, Borough')])
+    def test_well_formatted(self, inname, outname):
+        assert self.run_sanitizer_on({'tiger:county': inname})\
+            == [(outname, 'county', 'tiger')]
+
+
+    @pytest.mark.parametrize('name', ('Hamilton', 'Big, Road', ''))
+    def test_badly_formatted(self, name):
+        assert self.run_sanitizer_on({'tiger:county': name})\
+            == [(name, 'county', 'tiger')]
+
+
+    def test_unmatched(self):
+        assert self.run_sanitizer_on({'tiger:country': 'US'})\
+            == [('US', 'tiger', 'country')]

test/python/tools/test_refresh.py

@@ -17,6 +17,21 @@ def test_refresh_import_wikipedia_not_existing(dsn):
     assert refresh.import_wikipedia_articles(dsn, Path('.')) == 1
 
 
+def test_refresh_import_secondary_importance_non_existing(dsn):
+    assert refresh.import_secondary_importance(dsn, Path('.')) == 1
+
+def test_refresh_import_secondary_importance_testdb(dsn, src_dir, temp_db_conn, temp_db_cursor):
+    temp_db_cursor.execute('CREATE EXTENSION postgis')
+
+    if temp_db_conn.postgis_version_tuple()[0] < 3:
+        assert refresh.import_secondary_importance(dsn, src_dir / 'test' / 'testdb') > 0
+    else:
+        temp_db_cursor.execute('CREATE EXTENSION postgis_raster')
+        assert refresh.import_secondary_importance(dsn, src_dir / 'test' / 'testdb') == 0
+
+        assert temp_db_conn.table_exists('secondary_importance')
+
+
 @pytest.mark.parametrize("replace", (True, False))
 def test_refresh_import_wikipedia(dsn, src_dir, table_factory, temp_db_cursor, replace):
     if replace:
@@ -34,6 +49,7 @@ def test_recompute_importance(placex_table, table_factory, temp_db_conn, temp_db
     temp_db_cursor.execute("""CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE,
                                               country_code varchar(2),
                                               osm_type varchar(1), osm_id BIGINT,
+                                              centroid GEOMETRY,
                                               OUT importance FLOAT,
                                               OUT wikipedia TEXT)
                                AS $$ SELECT 0.1::float, 'foo'::text $$ LANGUAGE SQL""")

vagrant/Install-on-Ubuntu-18.sh

@@ -24,7 +24,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
     sudo apt install -y php-cgi
     sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
                         libboost-filesystem-dev libexpat1-dev zlib1g-dev\
-                        libbz2-dev libpq-dev \
+                        libbz2-dev libpq-dev liblua5.3-dev lua5.3\
                         postgresql-10-postgis-2.4 \
                         postgresql-contrib-10 postgresql-10-postgis-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-pip \

vagrant/Install-on-Ubuntu-20.sh

@@ -23,7 +23,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
     sudo apt install -y php-cgi
     sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
                         libboost-filesystem-dev libexpat1-dev zlib1g-dev \
-                        libbz2-dev libpq-dev \
+                        libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
                         postgresql-12-postgis-3 \
                         postgresql-contrib-12 postgresql-12-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \

vagrant/Install-on-Ubuntu-22.sh

@@ -23,7 +23,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
     sudo apt install -y php-cgi
     sudo apt install -y build-essential cmake g++ libboost-dev libboost-system-dev \
                         libboost-filesystem-dev libexpat1-dev zlib1g-dev \
-                        libbz2-dev libpq-dev \
+                        libbz2-dev libpq-dev liblua5.3-dev lua5.3 \
                         postgresql-server-dev-14 postgresql-14-postgis-3 \
                         postgresql-contrib-14 postgresql-14-postgis-3-scripts \
                         php-cli php-pgsql php-intl libicu-dev python3-dotenv \