prepare release 4.1.2

Special terms with operator name usually appear in combination with the
name. The current penalties only took name + special term into account
not special term + name.

Fixes #2876.

.github/FUNDING.yml

@@ -1,2 +0,0 @@
-github: lonvia
-custom: "https://nominatim.org/funding/"

.github/actions/build-nominatim/action.yml

@@ -9,10 +9,6 @@ inputs:
         description: 'Additional options to hand to cmake'
         required: false
         default: ''
-    lua:
-        description: 'Version of Lua to use'
-        required: false
-        default: '5.3'
 
 runs:
     using: "composite"
@@ -25,7 +21,7 @@ runs:
           shell: bash
         - name: Install prerequisites
           run: |
-            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION}
+            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev
             if [ "x$UBUNTUVER" == "x18" ]; then
                 pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 datrie
             else
@@ -35,7 +31,6 @@ runs:
           env:
             UBUNTUVER: ${{ inputs.ubuntu }}
             CMAKE_ARGS: ${{ inputs.cmake-args }}
-            LUA_VERSION: ${{ inputs.lua }}
 
         - name: Configure
           run: mkdir build && cd build && cmake $CMAKE_ARGS ../Nominatim

.github/workflows/ci-tests.yml

@@ -50,7 +50,7 @@ jobs:
                       pytest: py.test-3
                       php: 7.4
                     - ubuntu: 22
-                      postgresql: 15
+                      postgresql: 14
                       postgis: 3
                       pytest: py.test-3
                       php: 8.1
@@ -265,10 +265,6 @@ jobs:
               run: nominatim --version
               working-directory: /home/nominatim/nominatim-project
 
-            - name: Collect host OS information
-              run: nominatim admin --collect-os-info
-              working-directory: /home/nominatim/nominatim-project
-              
             - name: Import
               run: nominatim import --osm-file ../test.pbf
               working-directory: /home/nominatim/nominatim-project

.pylintrc

@@ -15,4 +15,4 @@ ignored-classes=NominatimArgs,closing
 #   typed Python is enabled. See also https://github.com/PyCQA/pylint/issues/5273
 disable=too-few-public-methods,duplicate-code,too-many-ancestors,bad-option-value,no-self-use,not-context-manager
 
-good-names=i,x,y,m,fd,db,cc
+good-names=i,x,y,fd,db,cc

CMakeLists.txt

@@ -19,8 +19,8 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 4)
-set(NOMINATIM_VERSION_MINOR 2)
+set(NOMINATIM_VERSION_MINOR 1)
-set(NOMINATIM_VERSION_PATCH 0)
+set(NOMINATIM_VERSION_PATCH 2)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 
@@ -63,6 +63,7 @@ if (BUILD_IMPORTER AND BUILD_OSM2PGSQL)
     endif()
     set(BUILD_TESTS_SAVED "${BUILD_TESTS}")
     set(BUILD_TESTS off)
+    set(WITH_LUA off CACHE BOOL "")
     add_subdirectory(osm2pgsql)
     set(BUILD_TESTS ${BUILD_TESTS_SAVED})
 endif()

CONTRIBUTING.md

@@ -64,39 +64,3 @@ Before submitting a pull request make sure that the tests pass:
   cd build
   make test
 ```
-
-## Releases
-
-Nominatim follows semantic versioning. Major releases are done for large changes
-that require (or at least strongly recommend) a reimport of the databases.
-Minor releases can usually be applied to exisiting databases Patch releases
-contain bug fixes only and are released from a separate branch where the
-relevant changes are cherry-picked from the master branch.
-
-Checklist for releases:
-
-* [ ] increase version in `nominatim/version.py` and CMakeLists.txt
-* [ ] update `ChangeLog` (copy information from patch releases from release branch)
-* [ ] complete `docs/admin/Migration.md`
-* [ ] update EOL dates in `SECURITY.md`
-* [ ] commit and make sure CI tests pass
-* [ ] test migration
-  * download, build and import previous version
-  * migrate using master version
-  * run updates using master version
-* [ ] prepare tarball:
-  * `git clone --recursive https://github.com/osm-search/Nominatim` (switch to right branch!)
-  * `rm -r .git* osm2pgsql/.git*`
-  * copy country data into `data/`
-  * add version to base directory and package
-* [ ] upload tarball to https://nominatim.org
-* [ ] prepare documentation
-  * check out new docs branch
-  * change git checkout instructions to tarball download instructions or adapt version on existing ones
-  * build documentation and copy to https://github.com/osm-search/nominatim-org-site
-  * add new version to history
-* [ ] check release tarball
-  * download tarball as per new documentation instructions
-  * compile and import Nominatim
-  * run `nominatim --version` to confirm correct version
-* [ ] tag new release and add a release on github.com

ChangeLog

@@ -1,15 +1,11 @@
-4.2.0
+4.1.2
+
+ * fix XSS vulnerability in debug view
+
+4.1.1
 
- * add experimental support for osm2pgsql flex style
- * introduce secondary importance value to be retrieved from a raster data file
-   (currently still unused, to replace address importance, thanks to @tareqpi)
- * add new report tool `nominatim admin --collect-os-info`
-   (thanks @micahcochran, @tareqpi)
- * reorganise index to improve lookup performance and size
- * run index creation after import in parallel
- * run ANALYZE more selectively to speed up continuation of indexing
  * fix crash on update when addr:interpolation receives an illegal value
- * fix minimum number of retrieved results to be at least 10
+ * fix minimum number of retrived results to be at least 10
  * fix search for combinations of special term + name (e.g Hotel Bellevue)
  * do not return interpolations without a parent street on reverse search
  * improve invalidation of linked places on updates
@@ -17,9 +13,8 @@
  * make sure socket timeouts are respected during replication
    (working around a bug in some versions of pyosmium)
  * update bundled osm2pgsql to 1.7.1
- * add support for PostgreSQL 15
  * typing fixes to work with latest type annotations from typeshed
- * smaller improvements to documentation (thanks to @mausch)
+ * smaller improvements to documention (thanks to @mausch)
 
 4.1.0
 
@@ -57,6 +52,10 @@
  * add setup instructions for updates and systemd
  * drop support for PostgreSQL 9.5
 
+4.0.2
+
+ * fix XSS vulnerability in debug view
+
 4.0.1
 
  * fix initialisation error in replication script
@@ -95,6 +94,10 @@
  * add testing of installation scripts via CI
  * drop support for Python < 3.6 and Postgresql < 9.5
 
+3.7.3
+
+ * fix XSS vulnerability in debug view
+
 3.7.2
 
  * fix database check for reverse-only imports

SECURITY.md

@@ -9,11 +9,10 @@ versions.
 
 | Version | End of support for security updates |
 | ------- | ----------------------------------- |
-| 4.2.x   | 2024-11-24                          |
-| 4.1.x   | 2024-08-05                          |
 | 4.0.x   | 2023-11-02                          |
 | 3.7.x   | 2023-04-05                          |
 | 3.6.x   | 2022-12-12                          |
+| 3.5.x   | 2022-06-05                          |
 
 ## Reporting a Vulnerability
 

docs/admin/Import.md

@@ -74,15 +74,15 @@ but it will improve the quality of the results if this is installed.
 This data is available as a binary download. Put it into your project directory:
 
     cd $PROJECT_DIR
-    wget https://nominatim.org/data/wikimedia-importance.sql.gz
+    wget https://www.nominatim.org/data/wikimedia-importance.sql.gz
 
 The file is about 400MB and adds around 4GB to the Nominatim database.
 
 !!! tip
-    If you forgot to download the wikipedia rankings, then you can
+    If you forgot to download the wikipedia rankings, you can also add
-    also add importances after the import. Download the SQL files, then
+    importances after the import. Download the files, then run
-    run `nominatim refresh --wiki-data --importance`. Updating
+    `nominatim refresh --wiki-data --importance`. Updating importances for
-    importances for a planet will take a couple of hours.
+    a planet can take a couple of hours.
 
 ### External postcodes
 
@@ -92,8 +92,8 @@ and the UK (using the [CodePoint OpenData set](https://osdatahub.os.uk/downloads
 This data can be optionally downloaded into the project directory:
 
     cd $PROJECT_DIR
-    wget https://nominatim.org/data/gb_postcodes.csv.gz
+    wget https://www.nominatim.org/data/gb_postcodes.csv.gz
-    wget https://nominatim.org/data/us_postcodes.csv.gz
+    wget https://www.nominatim.org/data/us_postcodes.csv.gz
 
 You can also add your own custom postcode sources, see
 [Customization of postcodes](../customize/Postcodes.md).

docs/admin/Installation.md

@@ -135,7 +135,7 @@ git clone --recursive https://github.com/openstreetmap/Nominatim.git
 The development version does not include the country grid. Download it separately:
 
 ```
-wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
+wget -O Nominatim/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
 ```
 
 ### Building Nominatim

docs/admin/Migration.md

@@ -15,14 +15,6 @@ breaking changes. **Please read them before running the migration.**
     If you are migrating from a version <3.6, then you still have to follow
     the manual migration steps up to 3.6.
 
-## 4.2.2 -> 4.2.3
-
-### Update interpolation functions
-
-When updating to this release, you need to run `nominatim refresh --functions`
-after updating and before restarting updates. Otherwise you may see an error
-`Splitting of Point geometries is unsupported` or similar.
-
 ## 4.0.0 -> 4.1.0
 
 ### ICU tokenizer is the new default

docs/api/Faq.md

@@ -59,27 +59,3 @@ suited for these kinds of queries.
 
 That said if you installed your own Nominatim instance you can use the
 `nominatim export` PHP script as basis to return such lists.
-
-#### 7. My result has a wrong postcode. Where does it come from?
-
-Most places in OSM don't have a postcode, so Nominatim tries to interpolate
-one. It first look at all the places that make up the address of the place.
-If one of them has a postcode defined, this is the one to be used. When
-none of the address parts has a postcode either, Nominatim interpolates one
-from the surrounding objects. If the postcode is for your result is one, then
-most of the time there is an OSM object with the wrong postcode nearby.
-
-To find the bad postcode, go to
-[https://nominatim.openstreetmap.org](https://nominatim.openstreetmap.org)
-and search for your place. When you have found it, click on the 'details' link
-under the result to go to the details page. There is a field 'Computed Postcode'
-which should display the bad postcode. Click on the 'how?' link. A small
-explanation text appears. It contains a link to a query for Overpass Turbo.
-Click on that and you get a map with all places in the area that have the bad
-postcode. If none is displayed, zoom the map out a bit and then click on 'Run'.
-
-Now go to [OpenStreetMap](https://openstreetmap.org) and fix the error you
-have just found. It will take at least a day for Nominatim to catch up with
-your data fix. Sometimes longer, depending on how much editing activity is in
-the area.
-

docs/api/Output.md

@@ -211,8 +211,8 @@ be more than one. The attributes of that element contain:
  * `ref` - content of `ref` tag if it exists
  * `lat`, `lon` - latitude and longitude of the centroid of the object
  * `boundingbox` - comma-separated list of corner coordinates ([see notes](#boundingbox))
- * `place_rank` - class [search rank](../customize/Ranking.md#search-rank)
+ * `place_rank` - class [search rank](../customize/Ranking#search-rank)
- * `address_rank` - place [address rank](../customize/Ranking.md#address-rank)
+ * `address_rank` - place [address rank](../customize/Ranking#address-rank)
  * `display_name` - full comma-separated address
  * `class`, `type` - key and value of the main OSM tag
  * `importance` - computed importance rank

docs/api/Reverse.md

@@ -35,7 +35,7 @@ Additional parameters are accepted as listed below.
 
 !!! warning "Deprecation warning"
     The reverse API used to allow address lookup for a single OSM object by
-    its OSM id. This use is now deprecated. Use the [Address Lookup API](Lookup.md)
+    its OSM id. This use is now deprecated. Use the [Address Lookup API](../Lookup)
     instead.
 
 ### Output format

docs/api/Status.md

@@ -58,10 +58,9 @@ code and message, e.g.
 Possible status codes are
 
    |     | message              | notes                                             |
-| --- | ------------------------------ | ----------------------------------------------------------------- |
+   |-----|----------------------|---------------------------------------------------|
    | 700 | "No database"        | connection failed                                 |
    | 701 | "Module failed"      | database could not load nominatim.so              |
    | 702 | "Module call failed" | nominatim.so loaded but calling a function failed |
    | 703 | "Query failed"       | test query against a database table failed        |
    | 704 | "No value"           | test query worked but returned no results         |
-| 705 | "Import date is not available" | No import dates were returned (enabling replication can fix this) |

docs/customize/Importance.md

@@ -1,49 +0,0 @@
-## Importance
-
-Search requests can yield multiple results which match equally well with
-the original query. In such case Nominatim needs to order the results
-according to a different criterion: importance. This is a measure for how
-likely it is that a user will search for a given place. This section explains
-the sources Nominatim uses for computing importance of a place and how to
-customize them.
-
-### How importance is computed
-
-The main value for importance is derived from page ranking values for Wikipedia
-pages for a place. For places that do not have their own
-Wikipedia page, a formula is used that derives a static importance from the
-places [search rank](../customize/Ranking.md#search-rank).
-
-In a second step, a secondary importance value is added which is meant to
-represent how well-known the general area is where the place is located. It
-functions as a tie-breaker between places with very similar primary
-importance values.
-
-nominatim.org has preprocessed importance tables for the
-[primary Wikipedia rankings](https://nominatim.org/data/wikimedia-importance.sql.gz)
-and for a secondary importance based on the number of tile views on openstreetmap.org.
-
-### Customizing secondary importance
-
-The secondary importance is implemented as a simple
-[Postgis raster](https://postgis.net/docs/raster.html) table, where Nominatim
-looks up the value for the coordinates of the centroid of a place. You can
-provide your own secondary importance raster in form of an SQL file named
-`secondary_importance.sql.gz` in your project directory.
-
-The SQL file needs to drop and (re)create a table `secondary_importance` which
-must as a minimum contain a column `rast` of type `raster`. The raster must
-be in EPSG:4326 and contain 16bit unsigned ints
-(`raster_constraint_pixel_types(rast) = '{16BUI}'). Any other columns in the
-table will be ignored. You must furthermore create an index as follows:
-
-```
-CREATE INDEX ON secondary_importance USING gist(ST_ConvexHull(gist))
-```
-
-The following raster2pgsql command will create a table that conforms to
-the requirements:
-
-```
-raster2pgsql -I -C -Y -d -t 128x128 input.tiff public.secondary_importance
-```

docs/customize/Settings.md

@@ -148,29 +148,6 @@ Setting this option to 'yes' means that Nominatim skips reindexing of contained
 objects when the area becomes too large.
 
 
-#### NOMINATIM_UPDATE_FORWARD_DEPENDENCIES
-
-| Summary            |                                                     |
-| --------------     | --------------------------------------------------- |
-| **Description:**   | Forward geometry changes to dependet objects |
-| **Format:**        | bool |
-| **Default:**       | no |
-| **Comment:**       | EXPERT ONLY. Must not be enabled after import. |
-
-The geometry of OSM ways and relations may change when a node that is part
-of the object is moved around. These changes are not propagated per default.
-The geometry of ways/relations is only updated the next time that the object
-itself is touched. When enabling this option, then dependent objects will
-be marked for update when one of its member objects changes.
-
-Enabling this option may slow down updates significantly.
-
-!!! warning
-    If you want to enable this option, it must be set already on import.
-    Do not enable this option on an existing database that was imported with
-    NOMINATIM_UPDATE_FORWARD_DEPENDENCIES=no.
-    Updates will become unusably slow.
-
 #### NOMINATIM_LANGUAGES
 
 | Summary            |                                                     |
@@ -666,7 +643,7 @@ The entries in the log file have the following format:
     <request time> <execution time in s> <number of results> <type> "<query string>"
 
 Request time is the time when the request was started. The execution time is
-given in seconds and corresponds to the time the query took executing in PHP.
+given in ms and corresponds to the time the query took executing in PHP.
 type contains the name of the endpoint used.
 
 Can be used as the same time as NOMINATIM_LOG_DB.

docs/customize/Tokenizers.md

@@ -213,15 +213,6 @@ The following is a list of sanitizers that are shipped with Nominatim.
     rendering:
         heading_level: 6
 
-##### clean-tiger-tags
-
-::: nominatim.tokenizer.sanitizers.clean_tiger_tags
-    selection:
-        members: False
-    rendering:
-        heading_level: 6
-
-
 
 #### Token Analysis
 

docs/extra.css

@@ -1,3 +1,7 @@
+.toctree-l3 {
+    display: none!important
+}
+
 table {
     margin-bottom: 12pt
 }

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Nominatim 4.2.4
+site_name: Nominatim Documentation
 theme: readthedocs
 docs_dir: ${CMAKE_CURRENT_BINARY_DIR}
 site_url: https://nominatim.org
@@ -30,7 +30,6 @@ nav:
         - 'Configuration Settings': 'customize/Settings.md'
         - 'Per-Country Data': 'customize/Country-Settings.md'
         - 'Place Ranking' : 'customize/Ranking.md'
-        - 'Importance' : 'customize/Importance.md'
         - 'Tokenizers' : 'customize/Tokenizers.md'
         - 'Special Phrases': 'customize/Special-Phrases.md'
         - 'External data: US housenumbers from TIGER': 'customize/Tiger.md'

lib-php/DebugHtml.php

@@ -135,7 +135,7 @@ class Debug
 
     public static function printSQL($sSQL)
     {
-        echo '<p><tt><font color="#aaa">'.$sSQL.'</font></tt></p>'."\n";
+        echo '<p><tt><font color="#aaa">'.htmlspecialchars($sSQL, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401).'</font></tt></p>'."\n";
     }
 
     private static function outputVar($mVar, $sPreNL)
@@ -178,11 +178,12 @@ class Debug
         }
 
         if (is_string($mVar)) {
-            echo "'$mVar'";
+            $sOut = "'$mVar'";
-            return strlen($mVar) + 2;
+        } else {
+            $sOut = (string)$mVar;
         }
 
-        echo (string)$mVar;
+        echo htmlspecialchars($sOut, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401);
-        return strlen((string)$mVar);
+        return strlen($sOut);
     }
 }

lib-php/PlaceLookup.php

@@ -187,12 +187,12 @@ class PlaceLookup
             return null;
         }
 
-        $aResults = $this->lookup(array($iPlaceID => new Result($iPlaceID)), 0, 30, true);
+        $aResults = $this->lookup(array($iPlaceID => new Result($iPlaceID)));
 
         return empty($aResults) ? null : reset($aResults);
     }
 
-    public function lookup($aResults, $iMinRank = 0, $iMaxRank = 30, $bAllowLinked = false)
+    public function lookup($aResults, $iMinRank = 0, $iMaxRank = 30)
     {
         Debug::newFunction('Place lookup');
 
@@ -247,9 +247,7 @@ class PlaceLookup
             if ($this->sAllowedTypesSQLList) {
                 $sSQL .= 'AND placex.class in '.$this->sAllowedTypesSQLList;
             }
-            if (!$bAllowLinked) {
             $sSQL .= '    AND linked_place_id is null ';
-            }
             $sSQL .= ' GROUP BY ';
             $sSQL .= '     osm_type, ';
             $sSQL .= '     osm_id, ';

lib-php/ReverseGeocode.php

@@ -189,16 +189,14 @@ class ReverseGeocode
         $sSQL .= '(select place_id, parent_place_id, rank_address, rank_search, country_code, geometry';
         $sSQL .= ' FROM placex';
         $sSQL .= ' WHERE ST_GeometryType(geometry) in (\'ST_Polygon\', \'ST_MultiPolygon\')';
-        // Ensure that query planner doesn't use the index on rank_search.
+        $sSQL .= ' AND rank_address Between 5 AND ' .$iMaxRank;
-        $sSQL .= ' AND coalesce(rank_search, 0) between 5 and ' .$iMaxRank;
-        $sSQL .= ' AND rank_address between 4 and 25'; // needed for index selection
         $sSQL .= ' AND geometry && '.$sPointSQL;
         $sSQL .= ' AND type != \'postcode\' ';
         $sSQL .= ' AND name is not null';
         $sSQL .= ' AND indexed_status = 0 and linked_place_id is null';
-        $sSQL .= ' ORDER BY rank_search DESC LIMIT 50 ) as a';
+        $sSQL .= ' ORDER BY rank_address DESC LIMIT 50 ) as a';
-        $sSQL .= ' WHERE ST_Contains(geometry, '.$sPointSQL.' )';
+        $sSQL .= ' WHERE ST_CONTAINS(geometry, '.$sPointSQL.' )';
-        $sSQL .= ' ORDER BY rank_search DESC LIMIT 1';
+        $sSQL .= ' ORDER BY rank_address DESC LIMIT 1';
         Debug::printSQL($sSQL);
 
         $aPoly = $this->oDB->getRow($sSQL, null, 'Could not determine polygon containing the point.');
@@ -210,7 +208,7 @@ class ReverseGeocode
             $iRankSearch = $aPoly['rank_search'];
             $iPlaceID = $aPoly['place_id'];
 
-            if ($iRankSearch != $iMaxRank) {
+            if ($iRankAddress != $iMaxRank) {
                 $sSQL = 'SELECT place_id FROM ';
                 $sSQL .= '(SELECT place_id, rank_search, country_code, geometry,';
                 $sSQL .= ' ST_distance('.$sPointSQL.', geometry) as distance';

lib-sql/functions/importance.sql

@@ -100,19 +100,14 @@ LANGUAGE plpgsql STABLE;
 
 CREATE OR REPLACE FUNCTION compute_importance(extratags HSTORE,
                                               country_code varchar(2),
-                                              rank_search SMALLINT,
+                                              osm_type varchar(1), osm_id BIGINT)
-                                              centroid GEOMETRY)
   RETURNS place_importance
   AS $$
 DECLARE
   match RECORD;
   result place_importance;
-  osm_views_exists BIGINT;
-  views BIGINT;
 BEGIN
-  -- add importance by wikipedia article if the place has one
+  FOR match IN SELECT * FROM get_wikipedia_match(extratags, country_code)
-  FOR match IN
-    SELECT * FROM get_wikipedia_match(extratags, country_code)
                WHERE language is not NULL
   LOOP
     result.importance := match.importance;
@@ -120,35 +115,17 @@ BEGIN
     RETURN result;
   END LOOP;
 
-  -- Nothing? Then try with the wikidata tag.
+  IF extratags ? 'wikidata' THEN
-  IF result.importance is null AND extratags ? 'wikidata' THEN
     FOR match IN SELECT * FROM wikipedia_article
                   WHERE wd_page_title = extratags->'wikidata'
-                  ORDER BY language = 'en' DESC, langcount DESC LIMIT 1
+                  ORDER BY language = 'en' DESC, langcount DESC LIMIT 1 LOOP
-    LOOP
       result.importance := match.importance;
       result.wikipedia := match.language || ':' || match.title;
       RETURN result;
     END LOOP;
   END IF;
 
-  -- Still nothing? Fall back to a default.
+  RETURN null;
-  IF result.importance is null THEN
-    result.importance := 0.75001 - (rank_search::float / 40);
-  END IF;
-
-{% if 'secondary_importance' in db.tables %}
-  FOR match IN
-    SELECT ST_Value(rast, centroid) as importance
-    FROM secondary_importance
-    WHERE ST_Intersects(ST_ConvexHull(rast), centroid) LIMIT 1
-  LOOP
-    -- Secondary importance as tie breaker with 0.0001 weight.
-    result.importance := result.importance + match.importance::float / 655350000;
-  END LOOP;
-{% endif %}
-
-  RETURN result;
 END;
 $$
 LANGUAGE plpgsql;

lib-sql/functions/interpolation.sql

@@ -52,9 +52,7 @@ BEGIN
 
   IF parent_place_id is null THEN
     FOR location IN SELECT place_id FROM placex
-        WHERE ST_DWithin(geom, placex.geometry, 0.001)
+        WHERE ST_DWithin(geom, placex.geometry, 0.001) and placex.rank_search = 26
-              and placex.rank_search = 26
-              and placex.osm_type = 'W' -- needed for index selection
         ORDER BY CASE WHEN ST_GeometryType(geom) = 'ST_Line' THEN
                   (ST_distance(placex.geometry, ST_LineInterpolatePoint(geom,0))+
                   ST_distance(placex.geometry, ST_LineInterpolatePoint(geom,0.5))+

lib-sql/functions/place_triggers.sql

@@ -34,11 +34,6 @@ BEGIN
     RETURN null;
   END IF;
 
-  -- Remove the place from the list of places to be deleted
-  DELETE FROM place_to_be_deleted pdel
-    WHERE pdel.osm_type = NEW.osm_type and pdel.osm_id = NEW.osm_id
-          and pdel.class = NEW.class;
-
   -- Have we already done this place?
   SELECT * INTO existing
     FROM place
@@ -47,6 +42,8 @@ BEGIN
 
   {% if debug %}RAISE WARNING 'Existing: %',existing.osm_id;{% endif %}
 
+  -- Handle a place changing type by removing the old data.
+  -- (This trigger is executed BEFORE INSERT of the NEW tuple.)
   IF existing.osm_type IS NULL THEN
     DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class;
   END IF;
@@ -190,11 +187,15 @@ BEGIN
     END IF;
     {% endif %}
 
-    IF existingplacex.osm_type is not NULL THEN
+    IF existing.osm_type IS NOT NULL THEN
-      -- Mark any existing place for delete in the placex table
+      -- Pathological case caused by the triggerless copy into place during initial import
-      UPDATE placex SET indexed_status = 100
+      -- force delete even for large areas, it will be reinserted later
-        WHERE placex.osm_type = NEW.osm_type and placex.osm_id = NEW.osm_id
+      UPDATE place SET geometry = ST_SetSRID(ST_Point(0,0), 4326)
-              and placex.class = NEW.class and placex.type = NEW.type;
+        WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
+              and class = NEW.class and type = NEW.type;
+      DELETE FROM place
+        WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
+              and class = NEW.class and type = NEW.type;
     END IF;
 
     -- Process it as a new insertion
@@ -205,27 +206,6 @@ BEGIN
 
     {% if debug %}RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name;{% endif %}
 
-    IF existing.osm_type is not NULL THEN
-      -- If there is already an entry in place, just update that, if necessary.
-      IF coalesce(existing.name, ''::hstore) != coalesce(NEW.name, ''::hstore)
-         or coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore)
-         or coalesce(existing.extratags, ''::hstore) != coalesce(NEW.extratags, ''::hstore)
-         or coalesce(existing.admin_level, 15) != coalesce(NEW.admin_level, 15)
-         or existing.geometry::text != NEW.geometry::text
-      THEN
-        UPDATE place
-          SET name = NEW.name,
-              address = NEW.address,
-              extratags = NEW.extratags,
-              admin_level = NEW.admin_level,
-              geometry = NEW.geometry
-          WHERE osm_type = NEW.osm_type and osm_id = NEW.osm_id
-                and class = NEW.class and type = NEW.type;
-      END IF;
-
-      RETURN NULL;
-    END IF;
-
     RETURN NEW;
   END IF;
 
@@ -341,67 +321,35 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
+
 CREATE OR REPLACE FUNCTION place_delete()
   RETURNS TRIGGER
   AS $$
 DECLARE
-  deferred BOOLEAN;
+  has_rank BOOLEAN;
 BEGIN
-  {% if debug %}RAISE WARNING 'Delete for % % %/%', OLD.osm_type, OLD.osm_id, OLD.class, OLD.type;{% endif %}
 
-  deferred := ST_IsValid(OLD.geometry) and ST_Area(OLD.geometry) > 2;
+  {% if debug %}RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type;{% endif %}
-  IF deferred THEN
+
-    SELECT bool_or(not (rank_address = 0 or rank_address > 25)) INTO deferred
+  -- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
-      FROM placex
+  IF st_area(OLD.geometry) > 2 and st_isvalid(OLD.geometry) THEN
-      WHERE osm_type = OLD.osm_type and osm_id = OLD.osm_id
+    SELECT bool_or(not (rank_address = 0 or rank_address > 25)) as ranked FROM placex WHERE osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type INTO has_rank;
-            and class = OLD.class and type = OLD.type;
+    IF has_rank THEN
+      insert into import_polygon_delete (osm_type, osm_id, class, type) values (OLD.osm_type,OLD.osm_id,OLD.class,OLD.type);
+      RETURN NULL;
+    END IF;
   END IF;
 
-  INSERT INTO place_to_be_deleted (osm_type, osm_id, class, type, deferred)
+  -- mark for delete
-    VALUES(OLD.osm_type, OLD.osm_id, OLD.class, OLD.type, deferred);
+  UPDATE placex set indexed_status = 100 where osm_type = OLD.osm_type and osm_id = OLD.osm_id and class = OLD.class and type = OLD.type;
 
-  RETURN NULL;
+  -- interpolations are special
+  IF OLD.osm_type='W' and OLD.class = 'place' and OLD.type = 'houses' THEN
+    UPDATE location_property_osmline set indexed_status = 100 where osm_id = OLD.osm_id; -- osm_id = wayid (=old.osm_id)
+  END IF;
+
+  RETURN OLD;
 END;
-$$ LANGUAGE plpgsql;
+$$
-
+LANGUAGE plpgsql;
-CREATE OR REPLACE FUNCTION flush_deleted_places()
-  RETURNS INTEGER
-  AS $$
-BEGIN
-  -- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
-  INSERT INTO import_polygon_delete (osm_type, osm_id, class, type)
-    SELECT osm_type, osm_id, class, type FROM place_to_be_deleted WHERE deferred;
-
-  -- delete from place table
-  ALTER TABLE place DISABLE TRIGGER place_before_delete;
-  DELETE FROM place USING place_to_be_deleted
-    WHERE place.osm_type = place_to_be_deleted.osm_type
-          and place.osm_id = place_to_be_deleted.osm_id
-          and place.class = place_to_be_deleted.class
-          and place.type = place_to_be_deleted.type
-          and not deferred;
-  ALTER TABLE place ENABLE TRIGGER place_before_delete;
-
-  -- Mark for delete in the placex table
-  UPDATE placex SET indexed_status = 100 FROM place_to_be_deleted
-    WHERE placex.osm_type = place_to_be_deleted.osm_type
-          and placex.osm_id = place_to_be_deleted.osm_id
-          and placex.class = place_to_be_deleted.class
-          and placex.type = place_to_be_deleted.type
-          and not deferred;
-
-   -- Mark for delete in interpolations
-   UPDATE location_property_osmline SET indexed_status = 100 FROM place_to_be_deleted
-    WHERE place_to_be_deleted.osm_type = 'W'
-          and place_to_be_deleted.class = 'place'
-          and place_to_be_deleted.type = 'houses'
-          and location_property_osmline.osm_id = place_to_be_deleted.osm_id
-          and not deferred;
-
-   -- Clear todo list.
-   TRUNCATE TABLE place_to_be_deleted;
-
-   RETURN NULL;
-END;
-$$ LANGUAGE plpgsql;
 

lib-sql/functions/placex_triggers.sql

@@ -107,17 +107,12 @@ LANGUAGE plpgsql STABLE;
 
 
 CREATE OR REPLACE FUNCTION find_associated_street(poi_osm_type CHAR(1),
-                                                  poi_osm_id BIGINT,
+                                                  poi_osm_id BIGINT)
-                                                  bbox GEOMETRY)
   RETURNS BIGINT
   AS $$
 DECLARE
   location RECORD;
   parent RECORD;
-  result BIGINT;
-  distance FLOAT;
-  new_distance FLOAT;
-  waygeom GEOMETRY;
 BEGIN
   FOR location IN
     SELECT members FROM planet_osm_rels
@@ -128,34 +123,19 @@ BEGIN
     FOR i IN 1..array_upper(location.members, 1) BY 2 LOOP
       IF location.members[i+1] = 'street' THEN
         FOR parent IN
-          SELECT place_id, geometry
+          SELECT place_id from placex
-           FROM placex
            WHERE osm_type = upper(substring(location.members[i], 1, 1))::char(1)
                  and osm_id = substring(location.members[i], 2)::bigint
                  and name is not null
                  and rank_search between 26 and 27
         LOOP
-          -- Find the closest 'street' member.
+          RETURN parent.place_id;
-          -- Avoid distance computation for the frequent case where there is
-          -- only one street member.
-          IF waygeom is null THEN
-            result := parent.place_id;
-            waygeom := parent.geometry;
-          ELSE
-            distance := coalesce(distance, ST_Distance(waygeom, bbox));
-            new_distance := ST_Distance(parent.geometry, bbox);
-            IF new_distance < distance THEN
-              distance := new_distance;
-              result := parent.place_id;
-              waygeom := parent.geometry;
-            END IF;
-          END IF;
         END LOOP;
       END IF;
     END LOOP;
   END LOOP;
 
-  RETURN result;
+  RETURN NULL;
 END;
 $$
 LANGUAGE plpgsql STABLE;
@@ -182,7 +162,7 @@ BEGIN
   {% if debug %}RAISE WARNING 'finding street for % %', poi_osm_type, poi_osm_id;{% endif %}
 
   -- Is this object part of an associatedStreet relation?
-  parent_place_id := find_associated_street(poi_osm_type, poi_osm_id, bbox);
+  parent_place_id := find_associated_street(poi_osm_type, poi_osm_id);
 
   IF parent_place_id is null THEN
     parent_place_id := find_parent_for_address(token_info, poi_partition, bbox);
@@ -205,7 +185,7 @@ BEGIN
         RETURN location.place_id;
       END IF;
 
-      parent_place_id := find_associated_street('W', location.osm_id, bbox);
+      parent_place_id := find_associated_street('W', location.osm_id);
     END LOOP;
   END IF;
 
@@ -217,7 +197,6 @@ BEGIN
         SELECT place_id FROM placex
          WHERE bbox && geometry AND _ST_Covers(geometry, ST_Centroid(bbox))
                AND rank_address between 5 and 25
-               AND ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon')
          ORDER BY rank_address desc
       LOOP
         RETURN location.place_id;
@@ -233,7 +212,6 @@ BEGIN
         SELECT place_id FROM placex
          WHERE bbox && geometry AND _ST_Covers(geometry, ST_Centroid(bbox))
                AND rank_address between 5 and 25
-               AND ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon')
         ORDER BY rank_address desc
       LOOP
         RETURN location.place_id;
@@ -297,9 +275,7 @@ BEGIN
 
   -- If extratags has a place tag, look for linked nodes by their place type.
   -- Area and node still have to have the same name.
-  IF bnd.extratags ? 'place' and bnd.extratags->'place' != 'postcode'
+  IF bnd.extratags ? 'place' and bnd_name is not null THEN
-     and bnd_name is not null
-  THEN
     FOR linked_placex IN
       SELECT * FROM placex
       WHERE (position(lower(name->'name') in bnd_name) > 0
@@ -308,6 +284,7 @@ BEGIN
         AND placex.osm_type = 'N'
         AND (placex.linked_place_id is null or placex.linked_place_id = bnd.place_id)
         AND placex.rank_search < 26 -- needed to select the right index
+        AND placex.type != 'postcode'
         AND ST_Covers(bnd.geometry, placex.geometry)
     LOOP
       {% if debug %}RAISE WARNING 'Found type-matching place node %', linked_placex.osm_id;{% endif %}
@@ -869,8 +846,7 @@ BEGIN
       FROM placex
       WHERE osm_type = 'R' and class = 'boundary' and type = 'administrative'
             and admin_level < NEW.admin_level and admin_level > 3
-            and rank_address between 1 and 25 -- for index selection
+            and rank_address > 0
-            and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') -- for index selection
             and geometry && NEW.centroid and _ST_Covers(geometry, NEW.centroid)
       ORDER BY admin_level desc LIMIT 1
     LOOP
@@ -898,9 +874,8 @@ BEGIN
           FROM placex,
                LATERAL compute_place_rank(country_code, 'A', class, type,
                                           admin_level, False, null) prank
-          WHERE class = 'place' and rank_address between 1 and 23
+          WHERE class = 'place' and rank_address < 24
                 and prank.address_rank >= NEW.rank_address
-                and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') -- select right index
                 and geometry && NEW.geometry
                 and geometry ~ NEW.geometry -- needed because ST_Relate does not do bbox cover test
                 and ST_Relate(geometry, NEW.geometry, 'T*T***FF*') -- contains but not equal
@@ -921,8 +896,6 @@ BEGIN
                LATERAL compute_place_rank(country_code, 'A', class, type,
                                           admin_level, False, null) prank
           WHERE prank.address_rank < 24
-                and rank_address between 1 and 25 -- select right index
-                and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') -- select right index
                 and prank.address_rank >= NEW.rank_address
                 and geometry && NEW.geometry
                 and geometry ~ NEW.geometry -- needed because ST_Relate does not do bbox cover test
@@ -943,8 +916,6 @@ BEGIN
              LATERAL compute_place_rank(country_code, 'A', class, type,
                                         admin_level, False, null) prank
         WHERE osm_type = 'R'
-              and rank_address between 1 and 25 -- select right index
-              and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') -- select right index
               and ((class = 'place' and prank.address_rank = NEW.rank_address)
                    or (class = 'boundary' and rank_address = NEW.rank_address))
               and geometry && NEW.centroid and _ST_Covers(geometry, NEW.centroid)
@@ -985,7 +956,7 @@ BEGIN
 
   NEW.importance := null;
   SELECT wikipedia, importance
-    FROM compute_importance(NEW.extratags, NEW.country_code, NEW.rank_search, NEW.centroid)
+    FROM compute_importance(NEW.extratags, NEW.country_code, NEW.osm_type, NEW.osm_id)
     INTO NEW.wikipedia,NEW.importance;
 
 {% if debug %}RAISE WARNING 'Importance computed from wikipedia: %', NEW.importance;{% endif %}
@@ -1067,7 +1038,7 @@ BEGIN
   IF linked_place is not null THEN
     -- Recompute the ranks here as the ones from the linked place might
     -- have been shifted to accommodate surrounding boundaries.
-    SELECT place_id, osm_id, class, type, extratags, rank_search,
+    SELECT place_id, osm_id, class, type, extratags,
            centroid, geometry,
            (compute_place_rank(country_code, osm_type, class, type, admin_level,
                               (extratags->'capital') = 'yes', null)).*
@@ -1108,7 +1079,7 @@ BEGIN
 
     SELECT wikipedia, importance
       FROM compute_importance(location.extratags, NEW.country_code,
-                              location.rank_search, NEW.centroid)
+                              'N', location.osm_id)
       INTO linked_wikipedia,linked_importance;
 
     -- Use the maximum importance if one could be computed from the linked object.

lib-sql/indices.sql

@@ -10,86 +10,65 @@
 
 CREATE INDEX IF NOT EXISTS idx_place_addressline_address_place_id
   ON place_addressline USING BTREE (address_place_id) {{db.tablespace.search_index}};
----
+
 CREATE INDEX IF NOT EXISTS idx_placex_rank_search
   ON placex USING BTREE (rank_search) {{db.tablespace.search_index}};
----
+
 CREATE INDEX IF NOT EXISTS idx_placex_rank_address
   ON placex USING BTREE (rank_address) {{db.tablespace.search_index}};
----
+
 CREATE INDEX IF NOT EXISTS idx_placex_parent_place_id
   ON placex USING BTREE (parent_place_id) {{db.tablespace.search_index}}
   WHERE parent_place_id IS NOT NULL;
----
+
-CREATE INDEX IF NOT EXISTS idx_placex_geometry ON placex
-  USING GIST (geometry) {{db.tablespace.search_index}};
----
 CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPolygon
   ON placex USING gist (geometry) {{db.tablespace.search_index}}
   WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')
     AND rank_address between 4 and 25 AND type != 'postcode'
     AND name is not null AND indexed_status = 0 AND linked_place_id is null;
----
+
 CREATE INDEX IF NOT EXISTS idx_osmline_parent_place_id
   ON location_property_osmline USING BTREE (parent_place_id) {{db.tablespace.search_index}}
   WHERE parent_place_id is not null;
----
+
 CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id
   ON location_property_osmline USING BTREE (osm_id) {{db.tablespace.search_index}};
----
+
 CREATE INDEX IF NOT EXISTS idx_postcode_postcode
   ON location_postcode USING BTREE (postcode) {{db.tablespace.search_index}};
 
-{% if drop %}
----
-  DROP INDEX IF EXISTS idx_placex_geometry_address_area_candidates;
-  DROP INDEX IF EXISTS idx_placex_geometry_buildings;
-  DROP INDEX IF EXISTS idx_placex_geometry_lower_rank_ways;
-  DROP INDEX IF EXISTS idx_placex_wikidata;
-  DROP INDEX IF EXISTS idx_placex_rank_address_sector;
-  DROP INDEX IF EXISTS idx_placex_rank_boundaries_sector;
-{% else %}
 -- Indices only needed for updating.
----
+
+{% if not drop %}
+  CREATE INDEX IF NOT EXISTS idx_placex_pendingsector
+    ON placex USING BTREE (rank_address,geometry_sector) {{db.tablespace.address_index}}
+    WHERE indexed_status > 0;
+
   CREATE INDEX IF NOT EXISTS idx_location_area_country_place_id
     ON location_area_country USING BTREE (place_id) {{db.tablespace.address_index}};
----
+
   CREATE UNIQUE INDEX IF NOT EXISTS idx_place_osm_unique
     ON place USING btree(osm_id, osm_type, class, type) {{db.tablespace.address_index}};
----
--- Table needed for running updates with osm2pgsql on place.
-  CREATE TABLE IF NOT EXISTS place_to_be_deleted (
-    osm_type CHAR(1),
-    osm_id BIGINT,
-    class TEXT,
-    type TEXT,
-    deferred BOOLEAN
-   );
 {% endif %}
 
 -- Indices only needed for search.
+
 {% if 'search_name' in db.tables %}
----
   CREATE INDEX IF NOT EXISTS idx_search_name_nameaddress_vector
     ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
----
   CREATE INDEX IF NOT EXISTS idx_search_name_name_vector
     ON search_name USING GIN (name_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
----
   CREATE INDEX IF NOT EXISTS idx_search_name_centroid
     ON search_name USING GIST (centroid) {{db.tablespace.search_index}};
 
   {% if postgres.has_index_non_key_column %}
----
     CREATE INDEX IF NOT EXISTS idx_placex_housenumber
       ON placex USING btree (parent_place_id)
       INCLUDE (housenumber) {{db.tablespace.search_index}}
       WHERE housenumber is not null;
----
     CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id_with_hnr
       ON location_property_osmline USING btree(parent_place_id)
       INCLUDE (startnumber, endnumber) {{db.tablespace.search_index}}
       WHERE startnumber is not null;
   {% endif %}
-
 {% endif %}

lib-sql/tables.sql

@@ -137,9 +137,7 @@ CREATE TABLE place_addressline (
   ) {{db.tablespace.search_data}};
 CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {{db.tablespace.search_index}};
 
----------  PLACEX - storage for all indexed places -----------------
+drop table if exists placex;
-
-DROP TABLE IF EXISTS placex;
 CREATE TABLE placex (
   place_id BIGINT NOT NULL,
   parent_place_id BIGINT,
@@ -159,66 +157,20 @@ CREATE TABLE placex (
   postcode TEXT,
   centroid GEOMETRY(Geometry, 4326)
   ) {{db.tablespace.search_data}};
-
 CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {{db.tablespace.search_index}};
-{% for osm_type in ('N', 'W', 'R') %}
+CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {{db.tablespace.search_index}};
-CREATE INDEX idx_placex_osmid_{{osm_type | lower}} ON placex
+CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {{db.tablespace.address_index}} WHERE linked_place_id IS NOT NULL;
-  USING BTREE (osm_id) {{db.tablespace.search_index}}
+CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}};
-  WHERE osm_type = '{{osm_type}}';
+CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {{db.tablespace.search_index}};
-{% endfor %}
-
--- Usage: - removing linkage status on update
---        - lookup linked places for /details
-CREATE INDEX idx_placex_linked_place_id ON placex
-  USING BTREE (linked_place_id) {{db.tablespace.address_index}}
-  WHERE linked_place_id IS NOT NULL;
-
--- Usage: - check that admin boundaries do not overtake each other rank-wise
---        - check that place node in a admin boundary with the same address level
---        - boundary is not completely contained in a place area
---        - parenting of large-area or unparentable features
-CREATE INDEX idx_placex_geometry_address_area_candidates ON placex
-  USING gist (geometry) {{db.tablespace.address_index}}
-  WHERE rank_address between 1 and 25
-        and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon');
-
--- Usage: - POI is within building with housenumber
 CREATE INDEX idx_placex_geometry_buildings ON placex
-  USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.address_index}}
+  USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.search_index}}
   WHERE address is not null and rank_search = 30
         and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon');
-
--- Usage: - linking of similar named places to boundaries
---        - linking of place nodes with same type to boundaries
---        - lookupPolygon()
 CREATE INDEX idx_placex_geometry_placenode ON placex
-  USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.address_index}}
+  USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.search_index}}
   WHERE osm_type = 'N' and rank_search < 26
-        and class = 'place' and type != 'postcode';
+        and class = 'place' and type != 'postcode' and linked_place_id is null;
-
+CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26;
--- Usage: - is node part of a way?
---        - find parent of interpolation spatially
-CREATE INDEX idx_placex_geometry_lower_rank_ways ON placex
-  USING {{postgres.spgist_geom}} (geometry) {{db.tablespace.address_index}}
-  WHERE osm_type = 'W' and rank_search >= 26;
-
--- Usage: - linking place nodes by wikidata tag to boundaries
-CREATE INDEX idx_placex_wikidata on placex
-  USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}}
-  WHERE extratags ? 'wikidata' and class = 'place'
-        and osm_type = 'N' and rank_search < 26;
-
--- The following two indexes function as a todo list for indexing.
-
-CREATE INDEX idx_placex_rank_address_sector ON placex
-  USING BTREE (rank_address, geometry_sector) {{db.tablespace.address_index}}
-  WHERE indexed_status > 0;
-
-CREATE INDEX idx_placex_rank_boundaries_sector ON placex
-  USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}}
-  WHERE class = 'boundary' and type = 'administrative'
-        and indexed_status > 0;
-
 
 DROP SEQUENCE IF EXISTS seq_place;
 CREATE SEQUENCE seq_place start 1;
@@ -276,7 +228,7 @@ CREATE SEQUENCE file start 1;
 
 -- null table so it won't error
 -- deliberately no drop - importing the table is expensive and static, if it is already there better to avoid removing it
-CREATE TABLE IF NOT EXISTS wikipedia_article (
+CREATE TABLE wikipedia_article (
     language text NOT NULL,
     title text NOT NULL,
     langcount integer,
@@ -290,12 +242,15 @@ CREATE TABLE IF NOT EXISTS wikipedia_article (
     wd_page_title text,
     instance_of text
 );
+ALTER TABLE ONLY wikipedia_article ADD CONSTRAINT wikipedia_article_pkey PRIMARY KEY (language, title);
+CREATE INDEX idx_wikipedia_article_osm_id ON wikipedia_article USING btree (osm_type, osm_id);
 
-CREATE TABLE IF NOT EXISTS wikipedia_redirect (
+CREATE TABLE wikipedia_redirect (
     language text,
     from_title text,
     to_title text
 );
+ALTER TABLE ONLY wikipedia_redirect ADD CONSTRAINT wikipedia_redirect_pkey PRIMARY KEY (language, from_title);
 
 -- osm2pgsql does not create indexes on the middle tables for Nominatim
 -- Add one for lookup of associated street relations.

module/CMakeLists.txt

@@ -1,6 +1,6 @@
 # just use the pgxs makefile
 
-foreach(suffix ${PostgreSQL_ADDITIONAL_VERSIONS} "15" "14" "13" "12" "11" "10" "9.6")
+foreach(suffix ${PostgreSQL_ADDITIONAL_VERSIONS} "14" "13" "12" "11" "10" "9.6")
     list(APPEND PG_CONFIG_HINTS
          "/usr/pgsql-${suffix}/bin")
 endforeach()

nominatim/clicmd/admin.py

@@ -20,7 +20,6 @@ from nominatim.clicmd.args import NominatimArgs
 
 LOG = logging.getLogger()
 
-
 class AdminFuncs:
     """\
     Analyse and maintain the database.
@@ -37,8 +36,6 @@ class AdminFuncs:
                           help='Migrate the database to a new software version')
         objs.add_argument('--analyse-indexing', action='store_true',
                           help='Print performance analysis of the indexing process')
-        objs.add_argument('--collect-os-info', action="store_true",
-                          help="Generate a report about the host system information")
         group = parser.add_argument_group('Arguments for cache warming')
         group.add_argument('--search-only', action='store_const', dest='target',
                            const='search',
@@ -73,14 +70,9 @@ class AdminFuncs:
             from ..tools import migration
             return migration.migrate(args.config, args)
 
-        if args.collect_os_info:
-            LOG.warning("Reporting System Information")
-            from ..tools import collect_os_info
-            collect_os_info.report_system_information(args.config)
-            return 0
-
         return 1
 
+
     def _warm(self, args: NominatimArgs) -> int:
         LOG.warning('Warming database caches')
         params = ['warm.php']

nominatim/clicmd/args.py

@@ -76,7 +76,6 @@ class NominatimArgs:
     warm: bool
     check_database: bool
     migrate: bool
-    collect_os_info: bool
     analyse_indexing: bool
     target: Optional[str]
     osm_id: Optional[str]
@@ -115,7 +114,6 @@ class NominatimArgs:
     address_levels: bool
     functions: bool
     wiki_data: bool
-    secondary_importance: bool
     importance: bool
     website: bool
     diffs: bool
@@ -184,10 +182,8 @@ class NominatimArgs:
         return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.osm2pgsql_path,
                     osm2pgsql_cache=self.osm2pgsql_cache or default_cache,
                     osm2pgsql_style=self.config.get_import_style_file(),
-                    osm2pgsql_style_path=self.config.config_dir,
                     threads=self.threads or default_threads,
                     dsn=self.config.get_libpq_dsn(),
-                    forward_dependencies=self.config.get_bool('UPDATE_FORWARD_DEPENDENCIES'),
                     flatnode_file=str(self.config.get_path('FLATNODE_FILE') or ''),
                     tablespaces=dict(slim_data=self.config.TABLESPACE_OSM_DATA,
                                      slim_index=self.config.TABLESPACE_OSM_INDEX,

nominatim/clicmd/refresh.py

@@ -63,8 +63,6 @@ class UpdateRefresh:
                            help='Update the PL/pgSQL functions in the database')
         group.add_argument('--wiki-data', action='store_true',
                            help='Update Wikipedia/data importance numbers')
-        group.add_argument('--secondary-importance', action='store_true',
-                           help='Update secondary importance raster data')
         group.add_argument('--importance', action='store_true',
                            help='Recompute place importances (expensive!)')
         group.add_argument('--website', action='store_true',
@@ -85,7 +83,7 @@ class UpdateRefresh:
                            help='Enable debug warning statements in functions')
 
 
-    def run(self, args: NominatimArgs) -> int: #pylint: disable=too-many-branches, too-many-statements
+    def run(self, args: NominatimArgs) -> int: #pylint: disable=too-many-branches
         from ..tools import refresh, postcodes
         from ..indexer.indexer import Indexer
 
@@ -117,20 +115,6 @@ class UpdateRefresh:
             with connect(args.config.get_libpq_dsn()) as conn:
                 refresh.load_address_levels_from_config(conn, args.config)
 
-        # Attention: must come BEFORE functions
-        if args.secondary_importance:
-            with connect(args.config.get_libpq_dsn()) as conn:
-                # If the table did not exist before, then the importance code
-                # needs to be enabled.
-                if not conn.table_exists('secondary_importance'):
-                    args.functions = True
-
-            LOG.warning('Import secondary importance raster data from %s', args.project_dir)
-            if refresh.import_secondary_importance(args.config.get_libpq_dsn(),
-                                                args.project_dir) > 0:
-                LOG.fatal('FATAL: Cannot update sendary importance raster data')
-                return 1
-
         if args.functions:
             LOG.warning('Create functions')
             with connect(args.config.get_libpq_dsn()) as conn:

nominatim/clicmd/setup.py

@@ -15,7 +15,7 @@ from pathlib import Path
 import psutil
 
 from nominatim.config import Configuration
-from nominatim.db.connection import connect
+from nominatim.db.connection import connect, Connection
 from nominatim.db import status, properties
 from nominatim.tokenizer.base import AbstractTokenizer
 from nominatim.version import version_str
@@ -72,8 +72,6 @@ class SetupAll:
         from ..tools import database_import, refresh, postcodes, freeze
         from ..indexer.indexer import Indexer
 
-        num_threads = args.threads or psutil.cpu_count() or 1
-
         country_info.setup_country_config(args.config)
 
         if args.continue_at is None:
@@ -96,21 +94,14 @@ class SetupAll:
                                             drop=args.no_updates,
                                             ignore_errors=args.ignore_errors)
 
+            self._setup_tables(args.config, args.reverse_only)
+
             LOG.warning('Importing wikipedia importance data')
             data_path = Path(args.config.WIKIPEDIA_DATA_PATH or args.project_dir)
             if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),
                                                  data_path) > 0:
                 LOG.error('Wikipedia importance dump file not found. '
-                          'Calculating importance values of locations will not '
+                          'Will be using default importances.')
-                          'use Wikipedia importance data.')
-
-            LOG.warning('Importing secondary importance raster data')
-            if refresh.import_secondary_importance(args.config.get_libpq_dsn(),
-                                                   args.project_dir) != 0:
-                LOG.error('Secondary importance file not imported. '
-                          'Falling back to default ranking.')
-
-            self._setup_tables(args.config, args.reverse_only)
 
         if args.continue_at is None or args.continue_at == 'load-data':
             LOG.warning('Initialise tables')
@@ -118,7 +109,8 @@ class SetupAll:
                 database_import.truncate_data_tables(conn)
 
             LOG.warning('Load data into placex table')
-            database_import.load_data(args.config.get_libpq_dsn(), num_threads)
+            database_import.load_data(args.config.get_libpq_dsn(),
+                                      args.threads or psutil.cpu_count() or 1)
 
         LOG.warning("Setting up tokenizer")
         tokenizer = self._get_tokenizer(args.continue_at, args.config)
@@ -129,15 +121,18 @@ class SetupAll:
                                        args.project_dir, tokenizer)
 
         if args.continue_at is None or args.continue_at in ('load-data', 'indexing'):
+            if args.continue_at is not None and args.continue_at != 'load-data':
+                with connect(args.config.get_libpq_dsn()) as conn:
+                    self._create_pending_index(conn, args.config.TABLESPACE_ADDRESS_INDEX)
             LOG.warning('Indexing places')
-            indexer = Indexer(args.config.get_libpq_dsn(), tokenizer, num_threads)
+            indexer = Indexer(args.config.get_libpq_dsn(), tokenizer,
+                              args.threads or psutil.cpu_count() or 1)
             indexer.index_full(analyse=not args.index_noanalyse)
 
         LOG.warning('Post-process tables')
         with connect(args.config.get_libpq_dsn()) as conn:
             database_import.create_search_indices(conn, args.config,
-                                                  drop=args.no_updates,
+                                                  drop=args.no_updates)
-                                                  threads=num_threads)
             LOG.warning('Create search index for default country names.')
             country_info.create_country_names(conn, tokenizer,
                                               args.config.get_str_list('LANGUAGES'))
@@ -193,6 +188,27 @@ class SetupAll:
         return tokenizer_factory.get_tokenizer_for_db(config)
 
 
+    def _create_pending_index(self, conn: Connection, tablespace: str) -> None:
+        """ Add a supporting index for finding places still to be indexed.
+
+            This index is normally created at the end of the import process
+            for later updates. When indexing was partially done, then this
+            index can greatly improve speed going through already indexed data.
+        """
+        if conn.index_exists('idx_placex_pendingsector'):
+            return
+
+        with conn.cursor() as cur:
+            LOG.warning('Creating support index')
+            if tablespace:
+                tablespace = 'TABLESPACE ' + tablespace
+            cur.execute(f"""CREATE INDEX idx_placex_pendingsector
+                            ON placex USING BTREE (rank_address,geometry_sector)
+                            {tablespace} WHERE indexed_status > 0
+                         """)
+        conn.commit()
+
+
     def _finalize_database(self, dsn: str, offline: bool) -> None:
         """ Determine the database date and set the status accordingly.
         """

nominatim/db/sql_preprocessor.py

@@ -11,7 +11,6 @@ from typing import Set, Dict, Any
 import jinja2
 
 from nominatim.db.connection import Connection
-from nominatim.db.async_connection import WorkerPool
 from nominatim.config import Configuration
 
 def _get_partitions(conn: Connection) -> Set[int]:
@@ -97,21 +96,3 @@ class SQLPreprocessor:
         with conn.cursor() as cur:
             cur.execute(sql)
         conn.commit()
-
-
-    def run_parallel_sql_file(self, dsn: str, name: str, num_threads: int = 1,
-                              **kwargs: Any) -> None:
-        """ Execure the given SQL files using parallel asynchronous connections.
-            The keyword arguments may supply additional parameters for
-            preprocessing.
-
-            After preprocessing the SQL code is cut at lines containing only
-            '---'. Each chunk is sent to one of the `num_threads` workers.
-        """
-        sql = self.env.get_template(name).render(**kwargs)
-
-        parts = sql.split('\n---\n')
-
-        with WorkerPool(dsn, num_threads) as pool:
-            for part in parts:
-                pool.next_free_worker().perform(part)

nominatim/indexer/indexer.py

@@ -128,64 +128,58 @@ class Indexer:
                     with conn.cursor() as cur:
                         cur.execute('ANALYZE')
 
-            if self.index_by_rank(0, 4) > 0:
+            self.index_by_rank(0, 4)
             _analyze()
 
-            if self.index_boundaries(0, 30) > 100:
+            self.index_boundaries(0, 30)
             _analyze()
 
-            if self.index_by_rank(5, 25) > 100:
+            self.index_by_rank(5, 25)
             _analyze()
 
-            if self.index_by_rank(26, 30) > 1000:
+            self.index_by_rank(26, 30)
             _analyze()
 
-            if self.index_postcodes() > 100:
+            self.index_postcodes()
             _analyze()
 
 
-    def index_boundaries(self, minrank: int, maxrank: int) -> int:
+    def index_boundaries(self, minrank: int, maxrank: int) -> None:
         """ Index only administrative boundaries within the given rank range.
         """
-        total = 0
         LOG.warning("Starting indexing boundaries using %s threads",
                     self.num_threads)
 
         with self.tokenizer.name_analyzer() as analyzer:
             for rank in range(max(minrank, 4), min(maxrank, 26)):
-                total += self._index(runners.BoundaryRunner(rank, analyzer))
+                self._index(runners.BoundaryRunner(rank, analyzer))
 
-        return total
+    def index_by_rank(self, minrank: int, maxrank: int) -> None:
-
-    def index_by_rank(self, minrank: int, maxrank: int) -> int:
         """ Index all entries of placex in the given rank range (inclusive)
             in order of their address rank.
 
             When rank 30 is requested then also interpolations and
             places with address rank 0 will be indexed.
         """
-        total = 0
         maxrank = min(maxrank, 30)
         LOG.warning("Starting indexing rank (%i to %i) using %i threads",
                     minrank, maxrank, self.num_threads)
 
         with self.tokenizer.name_analyzer() as analyzer:
             for rank in range(max(1, minrank), maxrank + 1):
-                total += self._index(runners.RankRunner(rank, analyzer), 20 if rank == 30 else 1)
+                self._index(runners.RankRunner(rank, analyzer), 20 if rank == 30 else 1)
 
             if maxrank == 30:
-                total += self._index(runners.RankRunner(0, analyzer))
+                self._index(runners.RankRunner(0, analyzer))
-                total += self._index(runners.InterpolationRunner(analyzer), 20)
+                self._index(runners.InterpolationRunner(analyzer), 20)
-
-        return total
 
 
-    def index_postcodes(self) -> int:
+    def index_postcodes(self) -> None:
         """Index the entries of the location_postcode table.
         """
         LOG.warning("Starting indexing postcodes using %s threads", self.num_threads)
 
-        return self._index(runners.PostcodeRunner(), 20)
+        self._index(runners.PostcodeRunner(), 20)
 
 
     def update_status_table(self) -> None:
@@ -197,7 +191,7 @@ class Indexer:
 
             conn.commit()
 
-    def _index(self, runner: runners.Runner, batch: int = 1) -> int:
+    def _index(self, runner: runners.Runner, batch: int = 1) -> None:
         """ Index a single rank or table. `runner` describes the SQL to use
             for indexing. `batch` describes the number of objects that
             should be processed with a single SQL statement
@@ -239,4 +233,4 @@ class Indexer:
 
                 conn.commit()
 
-        return progress.done()
+        progress.done()

nominatim/indexer/progress.py

@@ -55,7 +55,7 @@ class ProgressLogger:
 
         self.next_info += int(places_per_sec) * self.log_interval
 
-    def done(self) -> int:
+    def done(self) -> None:
         """ Print final statistics about the progress.
         """
         rank_end_time = datetime.now()
@@ -70,5 +70,3 @@ class ProgressLogger:
         LOG.warning("Done %d/%d in %d @ %.3f per second - FINISHED %s\n",
                     self.done_places, self.total_places, int(diff_seconds),
                     places_per_sec, self.name)
-
-        return self.done_places

nominatim/tokenizer/sanitizers/clean_tiger_tags.py

@@ -1,46 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# This file is part of Nominatim. (https://nominatim.org)
-#
-# Copyright (C) 2022 by the Nominatim developer community.
-# For a full list of authors see the git log.
-"""
-Sanitizer that preprocesses tags from the TIGER import.
-
-It makes the following changes:
-
-* remove state reference from tiger:county
-"""
-from typing import Callable
-import re
-
-from nominatim.tokenizer.sanitizers.base import ProcessInfo
-from nominatim.tokenizer.sanitizers.config import SanitizerConfig
-
-COUNTY_MATCH = re.compile('(.*), [A-Z][A-Z]')
-
-def _clean_tiger_county(obj: ProcessInfo) -> None:
-    """ Remove the state reference from tiger:county tags.
-
-        This transforms a name like 'Hamilton, AL' into 'Hamilton'.
-        If no state reference is detected at the end, the name is left as is.
-    """
-    if not obj.address:
-        return
-
-    for item in obj.address:
-        if item.kind == 'tiger' and item.suffix == 'county':
-            m = COUNTY_MATCH.fullmatch(item.name)
-            if m:
-                item.name = m[1]
-            # Switch kind and suffix, the split left them reversed.
-            item.kind = 'county'
-            item.suffix = 'tiger'
-
-            return
-
-
-def create(_: SanitizerConfig) -> Callable[[ProcessInfo], None]:
-    """ Create a housenumber processing function.
-    """
-    return _clean_tiger_county

nominatim/tools/check_database.py

@@ -114,10 +114,9 @@ def _get_indexes(conn: Connection) -> List[str]:
             indexes.extend(('idx_placex_housenumber',
                             'idx_osmline_parent_osm_id_with_hnr'))
     if conn.table_exists('place'):
-        indexes.extend(('idx_location_area_country_place_id',
+        indexes.extend(('idx_placex_pendingsector',
-                        'idx_place_osm_unique',
+                        'idx_location_area_country_place_id',
-                        'idx_placex_rank_address_sector',
+                        'idx_place_osm_unique'))
-                        'idx_placex_rank_boundaries_sector'))
 
     return indexes
 
@@ -200,7 +199,7 @@ def check_tokenizer(_: Connection, config: Configuration) -> CheckResult:
 def check_existance_wikipedia(conn: Connection, _: Configuration) -> CheckResult:
     """ Checking for wikipedia/wikidata data
     """
-    if not conn.table_exists('search_name') or not conn.table_exists('place'):
+    if not conn.table_exists('search_name'):
         return CheckState.NOT_APPLICABLE
 
     with conn.cursor() as cur:

nominatim/tools/collect_os_info.py

@@ -1,167 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# This file is part of Nominatim. (https://nominatim.org)
-#
-# Copyright (C) 2022 by the Nominatim developer community.
-# For a full list of authors see the git log.
-"""
-Collection of host system information including software versions, memory,
-storage, and database configuration.
-"""
-import os
-import subprocess
-import sys
-from pathlib import Path
-from typing import List, Optional, Tuple, Union, cast
-
-import psutil
-from psycopg2.extensions import make_dsn, parse_dsn
-
-from nominatim.config import Configuration
-from nominatim.db.connection import connect
-from nominatim.typing import DictCursorResults
-from nominatim.version import version_str
-
-
-def convert_version(ver_tup: Tuple[int, int]) -> str:
-    """converts tuple version (ver_tup) to a string representation"""
-    return ".".join(map(str, ver_tup))
-
-
-def friendly_memory_string(mem: float) -> str:
-    """Create a user friendly string for the amount of memory specified as mem"""
-    mem_magnitude = ("bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
-    mag = 0
-    # determine order of magnitude
-    while mem > 1000:
-        mem /= 1000
-        mag += 1
-
-    return f"{mem:.1f} {mem_magnitude[mag]}"
-
-
-def run_command(cmd: Union[str, List[str]]) -> str:
-    """Runs a command using the shell and returns the output from stdout"""
-    try:
-        if sys.version_info < (3, 7):
-            cap_out = subprocess.run(cmd, stdout=subprocess.PIPE, check=False)
-        else:
-            cap_out = subprocess.run(cmd, capture_output=True, check=False)
-        return cap_out.stdout.decode("utf-8")
-    except FileNotFoundError:
-        # non-Linux system should end up here
-        return f"Unknown (unable to find the '{cmd}' command)"
-
-
-def os_name_info() -> str:
-    """Obtain Operating System Name (and possibly the version)"""
-    os_info = None
-    # man page os-release(5) details meaning of the fields
-    if Path("/etc/os-release").is_file():
-        os_info = from_file_find_line_portion(
-            "/etc/os-release", "PRETTY_NAME", "=")
-    # alternative location
-    elif Path("/usr/lib/os-release").is_file():
-        os_info = from_file_find_line_portion(
-            "/usr/lib/os-release", "PRETTY_NAME", "="
-        )
-
-    # fallback on Python's os name
-    if os_info is None or os_info == "":
-        os_info = os.name
-
-    # if the above is insufficient, take a look at neofetch's approach to OS detection
-    return os_info
-
-
-# Note: Intended to be used on informational files like /proc
-def from_file_find_line_portion(
-    filename: str, start: str, sep: str, fieldnum: int = 1
-) -> Optional[str]:
-    """open filename, finds the line starting with the 'start' string.
-    Splits the line using seperator and returns a "fieldnum" from the split."""
-    with open(filename, encoding='utf8') as file:
-        result = ""
-        for line in file:
-            if line.startswith(start):
-                result = line.split(sep)[fieldnum].strip()
-        return result
-
-
-def get_postgresql_config(version: int) -> str:
-    """Retrieve postgres configuration file"""
-    try:
-        with open(f"/etc/postgresql/{version}/main/postgresql.conf", encoding='utf8') as file:
-            db_config = file.read()
-            file.close()
-            return db_config
-    except IOError:
-        return f"**Could not read '/etc/postgresql/{version}/main/postgresql.conf'**"
-
-
-def report_system_information(config: Configuration) -> None:
-    """Generate a report about the host system including software versions, memory,
-    storage, and database configuration."""
-
-    with connect(make_dsn(config.get_libpq_dsn(), dbname='postgres')) as conn:
-        postgresql_ver: str = convert_version(conn.server_version_tuple())
-
-        with conn.cursor() as cur:
-            cur.execute(f"""
-            SELECT datname FROM pg_catalog.pg_database 
-            WHERE datname='{parse_dsn(config.get_libpq_dsn())['dbname']}'""")
-            nominatim_db_exists = cast(Optional[DictCursorResults], cur.fetchall())
-            if nominatim_db_exists:
-                with connect(config.get_libpq_dsn()) as conn:
-                    postgis_ver: str = convert_version(conn.postgis_version_tuple())
-            else:
-                postgis_ver = "Unable to connect to database"
-
-    postgresql_config: str = get_postgresql_config(int(float(postgresql_ver)))
-
-    # Note: psutil.disk_partitions() is similar to run_command("lsblk")
-
-    # Note: run_command("systemd-detect-virt") only works on Linux, on other OSes
-    # should give a message: "Unknown (unable to find the 'systemd-detect-virt' command)"
-
-    # Generates the Markdown report.
-
-    report = f"""
-    **Instructions**
-    Use this information in your issue report at https://github.com/osm-search/Nominatim/issues
-    Redirect the output to a file:
-    $ ./collect_os_info.py > report.md
-
-
-    **Software Environment:**
-    - Python version: {sys.version}
-    - Nominatim version: {version_str()} 
-    - PostgreSQL version: {postgresql_ver} 
-    - PostGIS version: {postgis_ver}
-    - OS: {os_name_info()}
-    
-    
-    **Hardware Configuration:**
-    - RAM: {friendly_memory_string(psutil.virtual_memory().total)}
-    - number of CPUs: {psutil.cpu_count(logical=False)}
-    - bare metal/AWS/other cloud service (per systemd-detect-virt(1)): {run_command("systemd-detect-virt")} 
-    - type and size of disks:
-    **`df -h` - df - report file system disk space usage: **
-    ```
-    {run_command(["df", "-h"])}
-    ```
-    
-    **lsblk - list block devices: **
-    ```
-    {run_command("lsblk")}
-    ```
-    
-    
-    **Postgresql Configuration:**
-    ```
-    {postgresql_config}
-    ```
-    **Notes**
-    Please add any notes about anything above anything above that is incorrect.
-"""
-    print(report)

nominatim/tools/database_import.py

@@ -75,11 +75,6 @@ def setup_database_skeleton(dsn: str, rouser: Optional[str] = None) -> None:
         with conn.cursor() as cur:
             cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')
             cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')
-
-            postgis_version = conn.postgis_version_tuple()
-            if postgis_version[0] >= 3:
-                cur.execute('CREATE EXTENSION IF NOT EXISTS postgis_raster')
-
         conn.commit()
 
         _require_version('PostGIS',
@@ -230,8 +225,7 @@ def load_data(dsn: str, threads: int) -> None:
             cur.execute('ANALYSE')
 
 
-def create_search_indices(conn: Connection, config: Configuration,
+def create_search_indices(conn: Connection, config: Configuration, drop: bool = False) -> None:
-                          drop: bool = False, threads: int = 1) -> None:
     """ Create tables that have explicit partitioning.
     """
 
@@ -249,5 +243,4 @@ def create_search_indices(conn: Connection, config: Configuration,
 
     sql = SQLPreprocessor(conn, config)
 
-    sql.run_parallel_sql_file(config.get_libpq_dsn(),
+    sql.run_sql_file(conn, 'indices.sql', drop=drop)
-                              'indices.sql', min(8, threads), drop=drop)

nominatim/tools/exec_utils.py

@@ -10,7 +10,6 @@ Helper functions for executing external programs.
 from typing import Any, Union, Optional, Mapping, IO
 from pathlib import Path
 import logging
-import os
 import subprocess
 import urllib.request as urlrequest
 from urllib.parse import urlencode
@@ -117,27 +116,21 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None:
     env = get_pg_env(options['dsn'])
     cmd = [str(options['osm2pgsql']),
            '--hstore', '--latlon', '--slim',
+           '--with-forward-dependencies', 'false',
            '--log-progress', 'true',
            '--number-processes', str(options['threads']),
            '--cache', str(options['osm2pgsql_cache']),
+           '--output', 'gazetteer',
            '--style', str(options['osm2pgsql_style'])
           ]
-
+    if options['append']:
-    if str(options['osm2pgsql_style']).endswith('.lua'):
+        cmd.append('--append')
-        env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / 'flex-base.lua'),
-                                    os.environ.get('LUAPATH', ';')))
-        cmd.extend(('--output', 'flex'))
     else:
-        cmd.extend(('--output', 'gazetteer'))
+        cmd.append('--create')
-
-    cmd.append('--append' if options['append'] else '--create')
 
     if options['flatnode_file']:
         cmd.extend(('--flat-nodes', options['flatnode_file']))
 
-    if not options.get('forward_dependencies', False):
-        cmd.extend(('--with-forward-dependencies', 'false'))
-
     for key, param in (('slim_data', '--tablespace-slim-data'),
                        ('slim_index', '--tablespace-slim-index'),
                        ('main_data', '--tablespace-main-data'),

nominatim/tools/migration.py

@@ -315,36 +315,3 @@ def mark_internal_country_names(conn: Connection, config: Configuration, **_: An
                     names = {}
                 names['countrycode'] = country_code
                 analyzer.add_country_names(country_code, names)
-
-
-@_migration(4, 1, 99, 0)
-def add_place_deletion_todo_table(conn: Connection, **_: Any) -> None:
-    """ Add helper table for deleting data on updates.
-
-        The table is only necessary when updates are possible, i.e.
-        the database is not in freeze mode.
-    """
-    if conn.table_exists('place'):
-        with conn.cursor() as cur:
-            cur.execute("""CREATE TABLE IF NOT EXISTS place_to_be_deleted (
-                             osm_type CHAR(1),
-                             osm_id BIGINT,
-                             class TEXT,
-                             type TEXT,
-                             deferred BOOLEAN)""")
-
-
-@_migration(4, 1, 99, 1)
-def split_pending_index(conn: Connection, **_: Any) -> None:
-    """ Reorganise indexes for pending updates.
-    """
-    if conn.table_exists('place'):
-        with conn.cursor() as cur:
-            cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_rank_address_sector
-                           ON placex USING BTREE (rank_address, geometry_sector)
-                           WHERE indexed_status > 0""")
-            cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_rank_boundaries_sector
-                           ON placex USING BTREE (rank_search, geometry_sector)
-                           WHERE class = 'boundary' and type = 'administrative'
-                                 and indexed_status > 0""")
-            cur.execute("DROP INDEX IF EXISTS idx_placex_pendingsector")

nominatim/tools/refresh.py

@@ -15,7 +15,7 @@ from pathlib import Path
 from psycopg2 import sql as pysql
 
 from nominatim.config import Configuration
-from nominatim.db.connection import Connection, connect
+from nominatim.db.connection import Connection
 from nominatim.db.utils import execute_file
 from nominatim.db.sql_preprocessor import SQLPreprocessor
 from nominatim.version import version_str
@@ -146,25 +146,6 @@ def import_wikipedia_articles(dsn: str, data_path: Path, ignore_errors: bool = F
 
     return 0
 
-def import_secondary_importance(dsn: str, data_path: Path, ignore_errors: bool = False) -> int:
-    """ Replaces the secondary importance raster data table with new data.
-
-        Returns 0 if all was well and 1 if the raster SQL file could not
-        be found. Throws an exception if there was an error reading the file.
-    """
-    datafile = data_path / 'secondary_importance.sql.gz'
-    if not datafile.exists():
-        return 1
-
-    with connect(dsn) as conn:
-        postgis_version = conn.postgis_version_tuple()
-        if postgis_version[0] < 3:
-            LOG.error('PostGIS version is too old for using OSM raster data.')
-            return 2
-
-    execute_file(dsn, datafile, ignore_errors=ignore_errors)
-
-    return 0
 
 def recompute_importance(conn: Connection) -> None:
     """ Recompute wikipedia links and importance for all entries in placex.
@@ -176,7 +157,7 @@ def recompute_importance(conn: Connection) -> None:
         cur.execute("""
             UPDATE placex SET (wikipedia, importance) =
                (SELECT wikipedia, importance
-                FROM compute_importance(extratags, country_code, osm_type, osm_id, centroid))
+                FROM compute_importance(extratags, country_code, osm_type, osm_id))
             """)
         cur.execute("""
             UPDATE placex s SET wikipedia = d.wikipedia, importance = d.importance

nominatim/tools/replication.py

@@ -130,7 +130,10 @@ def update(conn: Connection, options: MutableMapping[str, Any],
         if endseq is None:
             return UpdateState.NO_CHANGES
 
-        run_osm2pgsql_updates(conn, options)
+        # Consume updates with osm2pgsql.
+        options['append'] = True
+        options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
+        run_osm2pgsql(options)
 
         # Write the current status to the file
         endstate = repl.get_state_info(endseq)
@@ -140,25 +143,6 @@ def update(conn: Connection, options: MutableMapping[str, Any],
     return UpdateState.UP_TO_DATE
 
 
-def run_osm2pgsql_updates(conn: Connection, options: MutableMapping[str, Any]) -> None:
-    """ Run osm2pgsql in append mode.
-    """
-    # Remove any stale deletion marks.
-    with conn.cursor() as cur:
-        cur.execute('TRUNCATE place_to_be_deleted')
-    conn.commit()
-
-    # Consume updates with osm2pgsql.
-    options['append'] = True
-    options['disable_jit'] = conn.server_version_tuple() >= (11, 0)
-    run_osm2pgsql(options)
-
-    # Handle deletions
-    with conn.cursor() as cur:
-        cur.execute('SELECT flush_deleted_places()')
-    conn.commit()
-
-
 def _make_replication_server(url: str, timeout: int) -> ContextManager[ReplicationServer]:
     """ Returns a ReplicationServer in form of a context manager.
 

nominatim/version.py

@@ -25,7 +25,7 @@ from typing import Optional, Tuple
 # patch level when cherry-picking the commit with the migration.
 #
 # Released versions always have a database patch level of 0.
-NOMINATIM_VERSION = (4, 2, 0, 0)
+NOMINATIM_VERSION = (4, 1, 2, 0)
 
 POSTGRESQL_REQUIRED_VERSION = (9, 6)
 POSTGIS_REQUIRED_VERSION = (2, 2)

settings/country-names/an.yaml

@@ -0,0 +1,71 @@
+name: 
+    default: De Nederlandse Antillen
+    af: Nederlandse Antille
+    an: Antillas Neerlandesas
+    ar: جزر الأنتيل
+    be: Нідэрландскія Антылы
+    bg: Холандски Антили
+    br: Antilhez Nederlandat
+    bs: Holandski Antili
+    ca: Antilles Neerlandeses
+    cs: Nizozemské Antily
+    cy: Antilles yr Iseldiroedd
+    da: Nederlandske Antiller
+    de: Niederländische Antillen
+    dv: ނެދަލޭންޑު އެންޓިލޭ
+    el: Ολλανδικές Αντίλλες
+    en: Netherlands Antilles
+    eo: Nederlandaj Antiloj
+    es: Antillas Neerlandesas;Antillas Holandesas;Indias Occidentales Holandesas
+    et: Hollandi Antillid
+    eu: Holandarren Antillak
+    fa: آنتیل هلند
+    fi: Alankomaiden Antillit
+    fo: Niðurlendsku Antillurnar
+    fr: Antilles néerlandaises
+    fy: Nederlânske Antillen
+    ga: Aintillí na hÍsiltíre
+    gl: Antillas Neerlandesas
+    he: האנטילים ההולנדיים
+    hi: नीदरलैंड एंटीलीज़
+    hr: Nizozemski Antili
+    hu: Holland Antillák
+    ia: Antillas Nederlandese
+    id: Antillen Belanda
+    io: Nederlandana Antili
+    is: Hollensku Antillaeyjar
+    it: Antille Olandesi
+    ja: オランダ領アンティル
+    jv: Antillen Walanda
+    ka: ნიდერლანდის ანტილები
+    kk: Антийлер
+    ko: 네덜란드령 안틸레스
+    kw: Antillys Iseldiryek
+    la: Antillae Nederlandiae
+    lb: Hollännesch Antillen
+    li: Nederlandse Antille
+    ln: Antiya ya Holanda
+    lt: Nyderlandų Antilai
+    lv: Antiļas
+    mn: Нидерландын Антиллийн Арлууд
+    mr: नेदरलँड्स अँटिल्स
+    ms: Antillen Belanda
+    nn: Dei nederlandske Antillane
+    "no": De nederlandske Antillene
+    pl: Antyle Holenderskie
+    pt: Antilhas Holandesas
+    ro: Antilele Olandeze
+    ru: Нидерландские Антилы
+    sh: Nizozemski Antili
+    sk: Holandské Antily
+    sl: Nizozemski Antili
+    sr: Холандски Антили
+    sv: Nederländska Antillerna
+    sw: Antili za Kiholanzi
+    ta: நெதர்லாந்து அண்டிலிசு
+    tg: Антил Ҳоланд
+    th: เนเธอร์แลนด์แอนทิลลิส
+    tr: Hollanda Antilleri
+    uk: Нідерландські Антильські острови
+    vi: Antille thuộc Hà Lan
+    zh: 荷属安的列斯

settings/country-names/aq.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Antarctica

settings/country-names/as.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: American Samoa

settings/country-names/aw.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Aruba

settings/country-names/ax.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Aland Islands

settings/country-names/bl.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Saint Barthélemy

settings/country-names/bq.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: "\N"

settings/country-names/bv.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Bouvet Island

settings/country-names/cc.yaml

@@ -0,0 +1,37 @@
+name: 
+    default: Cocos (Keeling) Islands
+    af: Cocos (Keeling) Eilande
+    ar: جزر كوكوس (كيلينغ)
+    be: Какосавыя (Кілінг) астравы
+    br: Inizi Kokoz
+    ca: Illes Cocos
+    da: Cocosøerne
+    de: Kokosinseln
+    el: Νησιά Κόκος
+    en: Cocos (Keeling) Islands
+    eo: Kokosinsuloj
+    es: Islas Cocos (Keeling)
+    et: Kookossaared
+    eu: Cocos (Keeling) uharteak
+    fa: جزایر کوکوس
+    fi: Kookossaaret
+    fr: Îles Cocos
+    fy: de Kokoseilannen
+    he: איי קוקוס (קילינג)
+    hr: Kokosovi otoci
+    hu: Kókusz (Keeling)-szigetek
+    id: Kepulauan Cocos (Keeling)
+    is: Kókoseyjar
+    it: Isole Cocos e Keeling
+    lt: Kokoso (Keelingo) salos
+    lv: Kokosu (Kīlinga) salas
+    mn: Кокосын (Кийлингийн) Арлууд
+    nl: Cocoseilanden
+    pl: Wyspy Kokosowe
+    ru: Кокосовые острова
+    sl: Kokosovi otoki
+    sv: Kokosöarna
+    tr: Cocos (Keeling) Adaları
+    uk: Кокосові острови
+    vi: Quần đảo Cocos (Keeling)
+    zh: 科科斯（基林）群島

settings/country-names/cw.yaml

@@ -0,0 +1,7 @@
+name: 
+    default: Curaçao
+    en: Curaçao
+    es: Curazao
+    fr: Curaçao
+    ru: Кюрасао
+    sv: Curaçao

settings/country-names/cx.yaml

@@ -0,0 +1,61 @@
+name: 
+    default: Christmas Island
+    af: Christmas-eiland
+    ar: جزيرة الميلاد
+    bg: Рождество
+    br: Enez Nedeleg
+    bs: Božićno ostrvo
+    ca: Illa Christmas
+    cs: Vánoční ostrov
+    cy: Ynys y Nadolig
+    da: Juleøen
+    de: Weihnachtsinsel
+    el: Νήσος των Χριστουγέννων
+    eo: Kristnaskinsulo
+    es: Isla de Navidad
+    et: Jõulusaar
+    eu: Christmas uhartea
+    fa: جزیره کریسمس
+    fi: Joulusaari
+    fr: Île Christmas
+    fy: Krysteilân
+    ga: Oileán na Nollag
+    gl: Illa de Nadal
+    he: טריטוריית האי חג המולד
+    hi: क्रिसमस आईलैंड
+    hr: Božićni otok
+    hu: Karácsony-sziget
+    id: Pulau Natal
+    is: Jólaeyja
+    it: Isola del Natale
+    ja: クリスマス島
+    ka: შობის კუნძული
+    kk: Кристмас аралы
+    ko: 크리스마스 섬
+    kw: Ynys Nadelik
+    lb: Chrëschtdagsinsel
+    lt: Kalėdų sala
+    lv: Ziemsvētku sala
+    mn: Зул Сарын Арал
+    mr: क्रिसमस द्वीप
+    ms: Pulau Krismas
+    nl: Christmaseiland
+    nn: Christmasøya
+    "no": Christmasøya
+    pl: Wyspa Bożego Narodzenia
+    pt: Ilha Christmas
+    ro: Insula Crăciunului
+    ru: Остров Рождества
+    sh: Božićni otok
+    sk: Vianočný ostrov
+    sl: Božični otoki
+    sr: Божићно Острво
+    sv: Julön
+    sw: Kisiwa cha Krismasi
+    ta: கிறிஸ்துமசு தீவு
+    th: เกาะคริสต์มาส
+    tr: Christmas Adası
+    uk: Острів Різдва
+    vi: Đảo Christmas
+    wo: Dunu Christmas
+    zh: 圣诞岛

settings/country-names/gf.yaml

@@ -0,0 +1,41 @@
+name: 
+    default: Guyane Française
+    af: Frans-Guyana
+    ar: غيانا
+    br: Gwiana c’hall
+    ca: Guaiana Francesa
+    cy: Guyane
+    da: Fransk Guyana
+    de: Französisch-Guayana
+    el: Γαλλική Γουιάνα
+    en: French Guiana
+    eo: Gujano
+    es: Guayana Francesa
+    et: Prantsuse Guajaana
+    fa: گویان فرانسه
+    fi: Ranskan Guayana
+    fr: Guyane française
+    fy: Frânsk Guyana
+    ga: Guáin na Fraince
+    gd: Guiana Fhrangach
+    he: גיאנה הצרפתית
+    hr: Francuska Gvajana
+    hu: Francia Guyana
+    id: Guyana Perancis
+    is: Franska Gvæjana
+    it: Guyana francese
+    la: Guiana Francica
+    li: Frans Guyana
+    lt: Prancūzijos Gviana
+    lv: Franču Gviāna
+    mn: Франц Гвиана
+    nl: Frans-Guyana
+    pl: Gujana Francuska
+    ru: Французская Гвиана
+    sl: Francoska Gvajana
+    sv: Franska Guyana
+    th: เฟรนช์เกียนา
+    tr: Fransız Guyanası
+    uk: Французька Гвіана
+    vi: Guyane thuộc Pháp
+    zh: 法属圭亚那

settings/country-names/gp.yaml

@@ -0,0 +1,31 @@
+name: 
+    default: Guadeloupe
+    ar: غوادلوب
+    be: Гвадэлупа
+    br: Gwadeloup
+    ca: Illa de Guadalupe
+    da: Guadeloupe
+    el: Γουαδελούπη
+    en: Guadeloupe
+    eo: Gvadelupo
+    es: Guadalupe
+    fa: گوادلوپ
+    fi: Guadeloupe
+    fr: Guadeloupe
+    fy: Guadelûp
+    ga: Guadalúip
+    he: גוואדלופ
+    hr: Gvadalupa
+    hu: Guadeloupe
+    is: Gvadelúpeyjar
+    it: Guadalupa
+    la: Guadalupa
+    lt: Gvadelupa
+    lv: Gvadelupa
+    mn: Гуаделупе
+    pl: Gwadelupa
+    ru: Гваделупа
+    sv: Guadeloupe
+    th: กวาเดอลูป
+    uk: Гваделупа
+    zh: 瓜德罗普

settings/country-names/gu.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Guam

settings/country-names/hk.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Hong Kong

settings/country-names/hm.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Heard Island and MaxDonald Islands

settings/country-names/mf.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Saint Martin

settings/country-names/mo.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Macao

settings/country-names/mp.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Northern Mariana Islands

settings/country-names/mq.yaml

@@ -0,0 +1,30 @@
+name: 
+    default: Martinique
+    ar: مارتينيك
+    be: Марцініка
+    br: Martinik
+    ca: Martinica
+    da: Martinique
+    el: Μαρτινίκα
+    en: Martinique
+    eo: Martiniko
+    es: Martinica
+    fa: مارتینیک
+    fi: Martinique
+    fr: Martinique
+    fy: Martinyk
+    he: מרטיניק
+    hr: Martinik
+    hu: Martinique
+    id: Martinik
+    is: Martinique
+    it: Martinica
+    la: Martinica
+    lt: Martinika
+    lv: Martinika
+    mn: Мартиник
+    pl: Martynika
+    ru: Мартиника
+    sv: Martinique
+    uk: Мартиніка
+    zh: 馬提尼克

settings/country-names/nc.yaml

@@ -0,0 +1,37 @@
+name: 
+    default: Nouvelle-Calédonie
+    af: Nieu-Caledonia
+    ar: كاليدونيا الجديدة
+    be: Новая Каледонія
+    br: Kaledonia Nevez
+    ca: Nova Caledònia
+    cy: Caledonia Newydd
+    da: Ny Kaledonien
+    de: Neukaledonien
+    el: Νέα Καληδονία
+    en: New Caledonia
+    eo: Nov-Kaledonio
+    es: Nueva Caledonia
+    fa: کالدونیای جدید
+    fi: Uusi-Kaledonia
+    fr: Nouvelle-Calédonie
+    ga: An Nua-Chaladóin
+    he: קלדוניה החדשה
+    hr: Nova Kaledonija
+    hu: Új-Kaledónia
+    id: Kaledonia Baru
+    is: Nýja-Kaledónía
+    it: Nuova Caledonia
+    la: Nova Caledonia
+    lt: Naujoji Kaledonija
+    lv: Jaunkaledonija
+    mn: Шинэ Каледони
+    nl: Nieuw-Caledonië
+    pl: Nowa Kaledonia
+    ru: Новая Каледония
+    sl: Nova Kaledonija
+    sv: Nya Kaledonien
+    th: นิวแคลิโดเนีย
+    tr: Yeni Kaledonya
+    uk: Нова Каледонія
+    zh: 新喀里多尼亚

settings/country-names/nf.yaml

@@ -0,0 +1,36 @@
+name: 
+    default: Norfolk Island
+    af: Norfolkeiland
+    ar: جزيرة نورفولك
+    be: Норфалк
+    br: Enez Norfolk
+    ca: Illa Norfolk
+    cy: Ynys Norfolk
+    da: Norfolk-øen
+    de: Norfolkinsel
+    en: Norfolk Island
+    eo: Norfolkinsulo
+    es: Isla Norfolk
+    et: Norfolki saar
+    fi: Norfolkinsaari
+    fr: Île Norfolk
+    fy: Norfolk
+    ga: Oileán Norfolk
+    he: האי נורפוק
+    hr: Otok Norfolk
+    hu: Norfolk-sziget
+    id: Pulau Norfolk
+    is: Norfolkeyja
+    it: Isola Norfolk
+    la: Insula Norfolcia
+    lt: Norfolko sala
+    lv: Norfolkas sala
+    mn: Норфолк Арал
+    nl: Norfolk
+    pl: Wyspa Norfolk
+    ru: Остров Норфолк
+    sv: Norfolkön
+    tr: Norfolk Adası
+    uk: Острів Норфолк
+    vi: Đảo Norfolk
+    zh: 诺福克岛

settings/country-names/pf.yaml

@@ -0,0 +1,77 @@
+name: 
+    default: Polynésie française
+    af: Franse Polynesië
+    an: Polinesia Franzesa
+    ar: بولونيزيا الفرنسية
+    az: Fransa Polineziyası
+    be: Французская Палінезія
+    bg: Френска Полинезия
+    br: Polinezia Frañs
+    bs: Francuska Polinezija
+    ca: Polinèsia Francesa
+    cs: Francouzská Polynésie
+    cy: Polynesia Ffrengig
+    da: Fransk Polynesien
+    de: Französisch-Polynesien
+    dv: ފަރަންސޭސި ޕޮލިނޭޝިއާ
+    el: Γαλλική Πολυνησία
+    en: French Polynesia
+    eo: Franca Polinezio
+    es: Polinesia Francesa
+    et: Prantsuse Polüneesia
+    eu: Frantziar Polinesia
+    fa: پلی‌نزی فرانسه
+    fi: Ranskan Polynesia
+    fr: Polynésie française
+    fy: Frânsk Polyneezje
+    ga: Polainéis na Fraince
+    gd: French Polynesia
+    gl: Polinesia francesa
+    he: פולינזיה הצרפתית
+    hi: फ्रेंच पोलीनेशिया
+    hr: Francuska Polinezija
+    hu: Francia Polinézia
+    id: Polinesia Perancis
+    io: Franca Polinezia
+    is: Franska Pólýnesía
+    it: Polinesia francese
+    ja: フランス領ポリネシア
+    jv: Polinesia Perancis
+    kk: Франция Полинезиясы
+    ko: 프랑스령 폴리네시아
+    kw: Polynesi Frynkek
+    la: Polynesia Francica
+    lb: Franséisch-Polynesien
+    lt: Prancūzijos Polinezija
+    lv: Franču Polinēzija
+    mi: Porinīhia Wīwī
+    mk: Француска Полинезија
+    mn: Францын Полинез
+    mr: फ्रेंच पॉलिनेशिया
+    ms: Polinesia Perancis
+    nl: Frans-Polynesië
+    nn: Fransk Polynesia
+    "no": Fransk Polynesia
+    oc: Polinesia Francesa
+    os: Францы Полинези
+    pl: Polinezja Francuska
+    pt: Polinésia Francesa
+    qu: Phransis Pulinisya
+    ro: Polinezia Franceză
+    ru: Французская Полинезия
+    se: Frankriikka Polynesia
+    sh: Francuska Polinezija
+    sk: Francúzska Polynézia
+    sl: Francoska Polinezija
+    sr: Француска Полинезија
+    sv: Franska Polynesien
+    sw: Polynesia ya Kifaransa
+    ta: பிரெஞ்சு பொலினீசியா
+    th: เฟรนช์โปลินีเซีย
+    tr: Fransız Polinezyası
+    ty: Pōrīnetia Farāni
+    ug: Fransiyige Qarashliq Polinéziye
+    uk: Французька Полінезія
+    vi: Polynésie thuộc Pháp
+    wo: Polineesi gu Faraas
+    zh: 法属波利尼西亚

settings/country-names/pm.yaml

@@ -0,0 +1,19 @@
+name: 
+    default: Saint-Pierre-et-Miquelon
+    af: Saint-Pierre et Miquelon
+    be: Святы П’ер і Міквелон
+    da: Saint Pierre og Miquelon
+    de: Saint-Pierre und Miquelon
+    en: Saint Pierre and Miquelon
+    eo: Sankta-Piero kaj Mikelono
+    es: San Pedro y Miguelón
+    fi: Saint-Pierre ja Miquelon
+    fr: Saint-Pierre-et-Miquelon
+    hr: Sveti Petar i Mikelon
+    hu: Saint-Pierre és Miquelon
+    lt: Sen Pjeras ir Mikelonas
+    lv: Senpjēra un Mikelona
+    mn: Сент Пьер ба Микелон
+    sv: Saint-Pierre och Miquelon
+    tr: Saint-Pierre ve Miquelon
+    uk: Сен-П'єр і Мікелон

settings/country-names/pr.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Puerto Rico

settings/country-names/re.yaml

@@ -0,0 +1,29 @@
+name: 
+    default: Réunion
+    af: Réunion
+    ar: ريونيون
+    be: Руньён
+    br: Ar Reunion
+    ca: Illa de la Reunió
+    da: Reunion
+    el: Ρεϊνιόν
+    eo: Reunio
+    es: La Reunión
+    fa: رئونیون
+    fi: Réunion
+    fr: La Réunion
+    he: ראוניון
+    hu: Réunion
+    is: Réunion
+    it: Riunione
+    la: Reunio
+    lt: Reunionas
+    lv: Reinjona
+    mn: Реюньон
+    pl: Reunion
+    ru: Реюньон
+    sl: Reunion
+    sv: Réunion
+    th: เรอูนียง
+    uk: Реюньйон
+    zh: 留尼汪

settings/country-names/sj.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Svalbard and Jan Mayen

settings/country-names/sx.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Sint Maarten

settings/country-names/tf.yaml

@@ -0,0 +1,48 @@
+name: 
+    default: Terres australes et antarctiques françaises
+    af: Franse Suidelike en Antarktiese Gebiede
+    an: Territorios Australs Franzeses
+    ar: الأراضي الجنوبية الفرنسية
+    be: Французскія Паўднёвыя тэрыторыі
+    bg: Френски южни и антарктически територии
+    br: Douaroù Aostral hag Antarktikel Frañs
+    ca: Terres Australs i Antàrtiques Franceses
+    cs: Francouzská jižní a antarktická území
+    da: Franske sydlige og Antarktiske territorier
+    de: Französische Süd- und Antarktisgebiete
+    el: Γαλλικά νότια και ανταρκτικά εδάφη
+    en: French Southern Lands
+    eo: Francaj Sudaj Teritorioj
+    es: Tierras Australes y Antárticas Francesas
+    eu: Frantziaren lurralde austral eta antartikoak
+    fi: Ranskan eteläiset ja antarktiset alueet
+    fr: Terres australes et antarctiques françaises
+    fy: Frânske Súdlike en Antarktyske Lannen
+    gl: Terras Austrais e Antárticas Francesas
+    hr: Francuski južni i antarktički teritoriji
+    hu: Francia déli és antarktiszi területek
+    id: Daratan Selatan dan Antarktika Perancis
+    is: Frönsku suðlægu landsvæðin
+    it: Terre Australi e Antartiche Francesi
+    ja: フランス領南方・南極地域
+    ko: 프랑스령 남부와 남극 지역
+    kw: Tiryow Deghow hag Antarktik Frynkek
+    lt: Prancūzijos Pietų Sritys
+    lv: Francijas Dienvidjūru un Antarktikas Zemes
+    nl: Franse Zuidelijke en Antarctische Gebieden
+    "no": De franske sørterritorier
+    oc: Tèrras Australas e Antarticas Francesas
+    pl: Francuskie Terytoria Południowe i Antarktyczne
+    pt: Terras Austrais e Antárticas Francesas
+    ro: Teritoriile australe şi antarctice franceze
+    ru: Французские Южные и Антарктические территории
+    sh: Francuske Južne Teritorije
+    sk: Francúzske južné a antarktické územia
+    sl: Francoske južne in antarktične dežele
+    sr: Француске јужне и антарктичке земље
+    sv: Franska sydterritorierna
+    ta: பிரெஞ்சு தென்னக நிலங்களும் அண்டாடிக் நிலமும்
+    tr: Fransız Güney ve Antarktika Toprakları
+    uk: Французькі Південні та Антарктичні території
+    vi: Vùng đất phía Nam và châu Nam Cực thuộc Pháp
+    zh: 法属南部领地

settings/country-names/um.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: United States Minor Outlying Islands

settings/country-names/vi.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: United States Virgin Islands

settings/country-names/wf.yaml

@@ -0,0 +1,68 @@
+name: 
+    default: Wallis-et-Futuna
+    af: Wallis-en-Futuna
+    an: Wallis e Futuna
+    ar: جزر واليس وفوتونا
+    be: Уоліс і Футуна
+    bg: Уолис и Футуна
+    br: Wallis ha Futuna
+    ca: Wallis i Futuna
+    cs: Wallis a Futuna
+    cy: Wallis a Futuna
+    da: Wallis og Futuna
+    de: Wallis und Futuna
+    dv: ވާލީ އަދި ފުތޫނާ
+    el: Ουώλλις και Φουτούνα
+    en: Wallis and Futuna Islands
+    eo: Valiso kaj Futuno
+    es: Wallis y Futuna
+    et: Wallis ja Futuna
+    eu: Wallis eta Futuna
+    fa: والیس و فوتونا
+    fi: Wallis- ja Futunasaaret
+    fr: Wallis-et-Futuna
+    fy: Wallis en Fûtûna
+    ga: Vailís agus Futúna
+    gl: Wallis e Futuna
+    he: ואליס ופוטונה
+    hr: Wallis i Futuna
+    hu: Wallis és Futuna
+    id: Wallis dan Futuna
+    io: Wallis e Futuna Insuli
+    is: Wallis- og Fútúnaeyjar
+    it: Wallis e Futuna
+    ja: ウォリス・フツナ
+    jv: Wallis lan Futuna
+    ko: 왈리스 퓌튀나
+    kw: Wallis ha Futuna
+    la: Vallis et Futuna
+    lb: Wallis a Futuna
+    lt: Walliso ir Futuna salos
+    lv: Volisa un Futuna
+    mn: Уоллис ба Футуна
+    mr: वालिस व फुतुना
+    ms: Wallis dan Futuna
+    nl: Wallis en Futuna
+    nn: Wallis- og Futunaøyane
+    "no": Wallis- og Futunaøyene
+    oc: Wallis e Futuna
+    pl: Wallis i Futuna
+    pt: Wallis e Futuna
+    ro: Wallis şi Futuna
+    ru: Уоллис и Футуна
+    se: Wallis ja Futuna
+    sh: Wallis i Futuna
+    sk: Wallis a Futuna
+    sl: Wallis in Futuna
+    sm: Wallis and Futuna
+    sr: Валис и Футуна
+    sv: Wallis- och Futunaöarna
+    sw: Wallis na Futuna
+    ta: வலிசும் புட்டூனாவும்
+    th: หมู่เกาะวาลลิสและหมู่เกาะฟุตูนา
+    tr: Wallis ve Futuna Adaları
+    ug: Wallis we Futuna Taqim Aralliri
+    uk: Волліс і Футуна
+    vi: Wallis và Futuna
+    wo: Wallis ak Futuna
+    zh: 瓦利斯和富图纳群岛

settings/country-names/yt.yaml

@@ -0,0 +1,2 @@
+name: 
+    default: Mayotte

settings/country_settings.yaml

@@ -61,6 +61,13 @@ am:
       pattern: "dddd"
 
 
+# Netherlands Antilles (De Nederlandse Antillen)
+an:
+    partition: 58
+    languages: nl, en, pap
+    names: !include country-names/an.yaml
+
+
 # Angola (Angola)
 ao:
     partition: 85
@@ -69,6 +76,14 @@ ao:
     postcode: no
 
 
+#  (Antarctica)
+aq:
+    partition: 181
+    languages: en, es, fr, ru
+    names: !include country-names/aq.yaml
+    postcode: no
+
+
 # Argentina (Argentina)
 ar:
     partition: 39
@@ -78,6 +93,13 @@ ar:
       pattern: "l?dddd(?:lll)?"
 
 
+#  (American Samoa)
+as:
+    partition: 182
+    languages: en, sm
+    names: !include country-names/as.yaml
+
+
 # Austria (Österreich)
 at:
     partition: 245
@@ -96,6 +118,21 @@ au:
       pattern: "dddd"
 
 
+#  (Aruba)
+aw:
+    partition: 183
+    languages: nl, pap
+    names: !include country-names/aw.yaml
+    postcode: no
+
+
+#  (Aland Islands)
+ax:
+    partition: 184
+    languages: sv
+    names: !include country-names/ax.yaml
+
+
 # Azerbaijan (Azərbaycan)
 az:
     partition: 119
@@ -184,6 +221,13 @@ bj:
     postcode: no
 
 
+#  (Saint Barthélemy)
+bl:
+    partition: 204
+    languages: fr
+    names: !include country-names/bl.yaml
+
+
 # Bermuda (Bermuda)
 bm:
     partition: 176
@@ -212,6 +256,13 @@ bo:
     postcode: no
 
 
+# Caribbean Netherlands (Caribisch Nederland)
+bq:
+    partition: 250
+    languages: nl
+    names: !include country-names/bq.yaml
+
+
 # Brazil (Brasil)
 br:
     partition: 121
@@ -239,6 +290,13 @@ bt:
       pattern: "ddddd"
 
 
+#  (Bouvet Island)
+bv:
+    partition: 185
+    languages: "no"
+    names: !include country-names/bv.yaml
+
+
 # Botswana (Botswana)
 bw:
     partition: 122
@@ -274,6 +332,13 @@ ca:
       output: \1 \2
 
 
+# Cocos (Keeling) Islands (Cocos (Keeling) Islands)
+cc:
+    partition: 118
+    languages: en
+    names: !include country-names/cc.yaml
+
+
 # Democratic Republic of the Congo (République démocratique du Congo)
 cd:
     partition: 229
@@ -385,6 +450,20 @@ cv:
       pattern: "dddd"
 
 
+# Curaçao (Curaçao)
+cw:
+    partition: 248
+    languages: nl, en
+    names: !include country-names/cw.yaml
+
+
+# Christmas Island (Christmas Island)
+cx:
+    partition: 177
+    languages: en
+    names: !include country-names/cx.yaml
+
+
 # Cyprus (Κύπρος - Kıbrıs)
 cy:
     partition: 114
@@ -604,6 +683,13 @@ ge:
       pattern: "dddd"
 
 
+# French Guiana (Guyane Française)
+gf:
+    partition: 231
+    languages: fr
+    names: !include country-names/gf.yaml
+
+
 # Guernsey (Guernsey)
 gg:
     partition: 77
@@ -659,6 +745,13 @@ gn:
       pattern: "ddd"
 
 
+# Guadeloupe (Guadeloupe)
+gp:
+    partition: 232
+    languages: fr
+    names: !include country-names/gp.yaml
+
+
 # Equatorial Guinea (Guinea Ecuatorial)
 gq:
     partition: 12
@@ -696,6 +789,13 @@ gt:
       pattern: "ddddd"
 
 
+# Guam (Guam)
+gu:
+    partition: 187
+    languages: en, ch
+    names: !include country-names/gu.yaml
+
+
 # Guinea-Bissau (Guiné-Bissau)
 gw:
     partition: 8
@@ -713,6 +813,20 @@ gy:
     postcode: no
 
 
+#  (Hong Kong)
+hk:
+    partition: 188
+    languages: zh-hant, en
+    names: !include country-names/hk.yaml
+
+
+#  (Heard Island and MaxDonald Islands)
+hm:
+    partition: 189
+    languages: en
+    names: !include country-names/hm.yaml
+
+
 # Honduras (Honduras)
 hn:
     partition: 56
@@ -1115,6 +1229,13 @@ me:
       pattern: "ddddd"
 
 
+# Saint Martin (Saint Martin)
+mf:
+    partition: 203
+    languages: fr
+    names: !include country-names/mf.yaml
+
+
 # Madagascar (Madagasikara)
 mg:
     partition: 164
@@ -1168,6 +1289,28 @@ mn:
       pattern: "ddddd"
 
 
+# Macao (Macao)
+mo:
+    partition: 191
+    languages: zh-hant, pt
+    names: !include country-names/mo.yaml
+    postcode: no
+
+
+# Northern Mariana Islands (Northern Mariana Islands)
+mp:
+    partition: 192
+    languages: ch, en
+    names: !include country-names/mp.yaml
+
+
+# Martinique (Martinique)
+mq:
+    partition: 233
+    languages: fr
+    names: !include country-names/mq.yaml
+
+
 # Mauritania (موريتانيا)
 mr:
     partition: 149
@@ -1255,6 +1398,13 @@ na:
       pattern: "ddddd"
 
 
+# New Caledonia (Nouvelle-Calédonie)
+nc:
+    partition: 234
+    languages: fr
+    names: !include country-names/nc.yaml
+
+
 # Niger (Niger)
 ne:
     partition: 226
@@ -1264,6 +1414,13 @@ ne:
       pattern: "dddd"
 
 
+# Norfolk Island (Norfolk Island)
+nf:
+    partition: 100
+    languages: en, pih
+    names: !include country-names/nf.yaml
+
+
 # Nigeria (Nigeria)
 ng:
     partition: 218
@@ -1362,6 +1519,13 @@ pe:
       pattern: "ddddd"
 
 
+# French Polynesia (Polynésie française)
+pf:
+    partition: 202
+    languages: fr
+    names: !include country-names/pf.yaml
+
+
 # Papua New Guinea (Papua Niugini)
 pg:
     partition: 71
@@ -1399,6 +1563,13 @@ pl:
       output: \1-\2
 
 
+# Saint Pierre and Miquelon (Saint-Pierre-et-Miquelon)
+pm:
+    partition: 236
+    languages: fr
+    names: !include country-names/pm.yaml
+
+
 # Pitcairn Islands (Pitcairn Islands)
 pn:
     partition: 113
@@ -1409,6 +1580,13 @@ pn:
       output: \1 \2
 
 
+# Puerto Rico (Puerto Rico)
+pr:
+    partition: 193
+    languages: es, en
+    names: !include country-names/pr.yaml
+
+
 # Palestinian Territory (Palestinian Territory)
 ps:
     partition: 194
@@ -1453,6 +1631,13 @@ qa:
     postcode: no
 
 
+#  (Réunion)
+re:
+    partition: 235
+    languages: fr
+    names: !include country-names/re.yaml
+
+
 # Romania (România)
 ro:
     partition: 170
@@ -1560,6 +1745,13 @@ si:
       pattern: "dddd"
 
 
+#  (Svalbard and Jan Mayen)
+sj:
+    partition: 197
+    languages: "no"
+    names: !include country-names/sj.yaml
+
+
 # Slovakia (Slovensko)
 sk:
     partition: 172
@@ -1639,6 +1831,13 @@ sv:
       pattern: "dddd"
 
 
+#  (Sint Maarten)
+sx:
+    partition: 249
+    languages: nl, en
+    names: !include country-names/sx.yaml
+
+
 # Syria (سوريا)
 sy:
     partition: 104
@@ -1674,6 +1873,13 @@ td:
     postcode: no
 
 
+# French Southern Lands (Terres australes et antarctiques françaises)
+tf:
+    partition: 132
+    languages: fr
+    names: !include country-names/tf.yaml
+
+
 # Togo (Togo)
 tg:
     partition: 243
@@ -1803,6 +2009,15 @@ ug:
     postcode: no
 
 
+#  (United States Minor Outlying Islands)
+um:
+    partition: 198
+    languages: en
+    names: !include country-names/um.yaml
+    postcode:
+      pattern: "96898"
+
+
 # United States (United States)
 us:
     partition: 2
@@ -1868,6 +2083,13 @@ vg:
       output: VG\1
 
 
+#  (United States Virgin Islands)
+vi:
+    partition: 199
+    languages: en
+    names: !include country-names/vi.yaml
+
+
 # Vietnam (Việt Nam)
 vn:
     partition: 75
@@ -1885,6 +2107,13 @@ vu:
     postcode: no
 
 
+# Wallis and Futuna Islands (Wallis-et-Futuna)
+wf:
+    partition: 238
+    languages: fr
+    names: !include country-names/wf.yaml
+
+
 # Samoa (Sāmoa)
 ws:
     partition: 131
@@ -1909,6 +2138,13 @@ ye:
     postcode: no
 
 
+# Mayotte (Mayotte)
+yt:
+    partition: 200
+    languages: fr
+    names: !include country-names/yt.yaml
+
+
 # South Africa (South Africa)
 za:
     partition: 76

settings/env.defaults

@@ -33,15 +33,6 @@ NOMINATIM_MAX_WORD_FREQUENCY=50000
 # If true, admin level changes on places with many contained children are blocked.
 NOMINATIM_LIMIT_REINDEXING=yes
 
-# When set to 'yes' changes to OSM objects will be propagated to dependent
-# objects. This means that geometries of way/relations are updated when a
-# node on a way or a way in a relation changes.
-# EXPERT ONLY: Use with care. Enabling this option might lead to significantly
-#              more load when updates are applied.
-#              Do not enable this option on an existing database.
-# The default is to not propagate these changes.
-NOMINATIM_UPDATE_FORWARD_DEPENDENCIES=no
-
 # Restrict search languages.
 # Normally Nominatim will include all language variants of name:XX
 # in the search index. Set this to a comma separated list of language

settings/flex-base.lua

@@ -1,383 +0,0 @@
--- Core functions for Nominatim import flex style.
---
-
-
--- The single place table.
-place_table = osm2pgsql.define_table{
-    name = "place",
-    ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
-    columns = {
-        { column = 'class', type = 'text', not_null = true },
-        { column = 'type', type = 'text', not_null = true },
-        { column = 'admin_level', type = 'smallint' },
-        { column = 'name', type = 'hstore' },
-        { column = 'address', type = 'hstore' },
-        { column = 'extratags', type = 'hstore' },
-        { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true },
-    }
-}
-
-------------- Place class ------------------------------------------
-
-local Place = {}
-Place.__index = Place
-
-function Place.new(object, geom_func)
-    local self = setmetatable({}, Place)
-    self.object = object
-    self.geom_func = geom_func
-
-    self.admin_level = tonumber(self.object:grab_tag('admin_level'))
-    if self.admin_level == nil
-       or self.admin_level <= 0 or self.admin_level > 15
-       or math.floor(self.admin_level) ~= self.admin_level then
-        self.admin_level = 15
-    end
-
-    self.num_entries = 0
-    self.has_name = false
-    self.names = {}
-    self.address = {}
-    self.extratags = {}
-
-    return self
-end
-
-function Place:delete(data)
-    if data.match ~= nil then
-        for k, v in pairs(self.object.tags) do
-            if data.match(k, v) then
-                self.object.tags[k] = nil
-            end
-        end
-    end
-end
-
-function Place:grab_extratags(data)
-    local count = 0
-
-    if data.match ~= nil then
-        for k, v in pairs(self.object.tags) do
-            if data.match(k, v) then
-                self.object.tags[k] = nil
-                self.extratags[k] = v
-                count = count + 1
-            end
-        end
-    end
-
-    return count
-end
-
-function Place:grab_address(data)
-    local count = 0
-
-    if data.match ~= nil then
-        for k, v in pairs(self.object.tags) do
-            if data.match(k, v) then
-                self.object.tags[k] = nil
-
-                if data.include_on_name == true then
-                    self.has_name = true
-                end
-
-                if data.out_key ~= nil then
-                    self.address[data.out_key] = v
-                    return 1
-                end
-
-                if k:sub(1, 5) == 'addr:' then
-                    self.address[k:sub(6)] = v
-                elseif k:sub(1, 6) == 'is_in:' then
-                    self.address[k:sub(7)] = v
-                else
-                    self.address[k] = v
-                end
-                count = count + 1
-            end
-        end
-    end
-
-    return count
-end
-
-function Place:set_address(key, value)
-    self.address[key] = value
-end
-
-function Place:grab_name(data)
-    local count = 0
-
-    if data.match ~= nil then
-        for k, v in pairs(self.object.tags) do
-            if data.match(k, v) then
-                self.object.tags[k] = nil
-                self.names[k] = v
-                if data.include_on_name ~= false then
-                    self.has_name = true
-                end
-                count = count + 1
-            end
-        end
-    end
-
-    return count
-end
-
-function Place:grab_tag(key)
-    return self.object:grab_tag(key)
-end
-
-function Place:tags()
-    return self.object.tags
-end
-
-function Place:write_place(k, v, mtype, save_extra_mains)
-    if mtype == nil then
-        return 0
-    end
-
-    v = v or self.object.tags[k]
-    if v == nil then
-        return 0
-    end
-
-    if type(mtype) == 'table' then
-        mtype = mtype[v] or mtype[1]
-    end
-
-    if mtype == 'always' or (self.has_name and mtype == 'named') then
-        return self:write_row(k, v, save_extra_mains)
-    end
-
-    if mtype == 'named_with_key' then
-        local names = {}
-        local prefix = k .. ':name'
-        for namek, namev in pairs(self.object.tags) do
-            if namek:sub(1, #prefix) == prefix
-               and (#namek == #prefix
-                    or namek:sub(#prefix + 1, #prefix + 1) == ':') then
-                names[namek:sub(#k + 2)] = namev
-            end
-        end
-
-        if next(names) ~= nil then
-            local saved_names = self.names
-            self.names = names
-
-            local results = self:write_row(k, v, save_extra_mains)
-
-            self.names = saved_names
-
-            return results
-        end
-    end
-
-    return 0
-end
-
-function Place:write_row(k, v, save_extra_mains)
-    if self.geometry == nil then
-        self.geometry = self.geom_func(self.object)
-    end
-    if self.geometry:is_null() then
-        return 0
-    end
-
-    if save_extra_mains then
-        for extra_k, extra_v in pairs(self.object.tags) do
-            if extra_k ~= k then
-                self.extratags[extra_k] = extra_v
-            end
-        end
-    end
-
-    place_table:insert{
-        class = k,
-        type = v,
-        admin_level = self.admin_level,
-        name = next(self.names) and self.names,
-        address = next(self.address) and self.address,
-        extratags = next(self.extratags) and self.extratags,
-        geometry = self.geometry
-    }
-
-    if save_extra_mains then
-        for k, v in pairs(self.object.tags) do
-            self.extratags[k] = nil
-        end
-    end
-
-    self.num_entries = self.num_entries + 1
-
-    return 1
-end
-
-
-function tag_match(data)
-    if data == nil or next(data) == nil then
-        return nil
-    end
-
-    local fullmatches = {}
-    local key_prefixes = {}
-    local key_suffixes = {}
-
-    if data.keys ~= nil then
-        for _, key in pairs(data.keys) do
-            if key:sub(1, 1) == '*' then
-                if #key > 1 then
-                    if key_suffixes[#key - 1] == nil then
-                        key_suffixes[#key - 1] = {}
-                    end
-                    key_suffixes[#key - 1][key:sub(2)] = true
-                end
-            elseif key:sub(#key, #key) == '*' then
-                if key_prefixes[#key - 1] == nil then
-                    key_prefixes[#key - 1] = {}
-                end
-                key_prefixes[#key - 1][key:sub(1, #key - 1)] = true
-            else
-                fullmatches[key] = true
-            end
-        end
-    end
-
-    if data.tags ~= nil then
-        for k, vlist in pairs(data.tags) do
-            if fullmatches[k] == nil then
-                fullmatches[k] = {}
-                for _, v in pairs(vlist) do
-                    fullmatches[k][v] = true
-                end
-            end
-        end
-    end
-
-    return function (k, v)
-        if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then
-            return true
-        end
-
-        for slen, slist in pairs(key_suffixes) do
-            if #k >= slen and slist[k:sub(-slen)] ~= nil then
-                return true
-            end
-        end
-
-        for slen, slist in pairs(key_prefixes) do
-            if #k >= slen and slist[k:sub(1, slen)] ~= nil then
-                return true
-            end
-        end
-
-        return false
-    end
-end
-
-
--- Process functions for all data types
-function osm2pgsql.process_node(object)
-
-    local function geom_func(o)
-        return o:as_point()
-    end
-
-    process_tags(Place.new(object, geom_func))
-end
-
-function osm2pgsql.process_way(object)
-
-    local function geom_func(o)
-        local geom = o:as_polygon()
-
-        if geom:is_null() then
-            geom = o:as_linestring()
-        end
-
-        return geom
-    end
-
-    process_tags(Place.new(object, geom_func))
-end
-
-function relation_as_multipolygon(o)
-    return o:as_multipolygon()
-end
-
-function relation_as_multiline(o)
-    return o:as_multilinestring():line_merge()
-end
-
-function osm2pgsql.process_relation(object)
-    local geom_func = RELATION_TYPES[object.tags.type]
-
-    if geom_func ~= nil then
-        process_tags(Place.new(object, geom_func))
-    end
-end
-
-function process_tags(o)
-    local fallback
-
-    o:delete{match = PRE_DELETE}
-    o:grab_extratags{match = PRE_EXTRAS}
-
-    -- Exception for boundary/place double tagging
-    if o.object.tags.boundary == 'administrative' then
-        o:grab_extratags{match = function (k, v)
-            return k == 'place' and v:sub(1,3) ~= 'isl'
-        end}
-    end
-
-    -- address keys
-    o:grab_address{match=COUNTRY_TAGS, out_key='country'}
-    if o.address.country ~= nil and #o.address.country ~= 2 then
-        o.address['country'] = nil
-    end
-    if o:grab_name{match=HOUSENAME_TAGS} > 0 then
-        fallback = {'place', 'house'}
-    end
-    if o:grab_address{match=HOUSENUMBER_TAGS, include_on_name = true} > 0 and fallback == nil then
-        fallback = {'place', 'house'}
-    end
-    if o:grab_address{match=POSTCODES, out_key='postcode'} > 0 and fallback == nil then
-        fallback = {'place', 'postcode'}
-    end
-
-    local is_interpolation = o:grab_address{match=INTERPOLATION_TAGS} > 0
-
-    o:grab_address{match=ADDRESS_TAGS}
-
-    if is_interpolation then
-        o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS)
-        return
-    end
-
-    -- name keys
-    o:grab_name{match = NAMES}
-    o:grab_name{match = REFS, include_on_name = false}
-
-    o:delete{match = POST_DELETE}
-    o:grab_extratags{match = POST_EXTRAS}
-
-    -- collect main keys
-    local num_mains = 0
-    for k, v in pairs(o:tags()) do
-        num_mains = num_mains + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS)
-    end
-
-    if num_mains == 0 then
-        for tag, mtype in pairs(MAIN_FALLBACK_KEYS) do
-            if o:write_place(tag, nil, mtype, SAVE_EXTRA_MAINS) > 0 then
-                return
-            end
-        end
-
-        if fallback ~= nil then
-            o:write_place(fallback[1], fallback[2], 'always', SAVE_EXTRA_MAINS)
-        end
-    end
-end
-
-

settings/icu_tokenizer.yaml

@@ -35,7 +35,6 @@ sanitizers:
     - step: clean-postcodes
       convert-to-address: yes
       default-pattern: "[A-Z0-9- ]{3,12}"
-    - step: clean-tiger-tags
     - step: split-name-list
     - step: strip-brace-terms
     - step: tag-analyzer-by-language

settings/import-extratags.lua

@@ -1,129 +0,0 @@
-require('flex-base')
-
-RELATION_TYPES = {
-    multipolygon = relation_as_multipolygon,
-    boundary = relation_as_multipolygon,
-    waterway = relation_as_multiline
-}
-
-MAIN_KEYS = {
-    emergency = 'always',
-    historic = 'always',
-    military = 'always',
-    natural = 'named',
-    landuse = 'named',
-    highway = {'always',
-               street_lamp = 'named',
-               traffic_signals = 'named',
-               service = 'named',
-               cycleway = 'named',
-               path = 'named',
-               footway = 'named',
-               steps = 'named',
-               bridleway = 'named',
-               track = 'named',
-               motorway_link = 'named',
-               trunk_link = 'named',
-               primary_link = 'named',
-               secondary_link = 'named',
-               tertiary_link = 'named'},
-    railway = 'named',
-    man_made = 'always',
-    aerialway = 'always',
-    boundary = {'named',
-                postal_code = 'named'},
-    aeroway = 'always',
-    amenity = 'always',
-    club = 'always',
-    craft = 'always',
-    leisure = 'always',
-    office = 'always',
-    mountain_pass = 'always',
-    shop = 'always',
-    tourism = 'always',
-    bridge = 'named_with_key',
-    tunnel = 'named_with_key',
-    waterway = 'named',
-    place = 'always'
-}
-
-MAIN_FALLBACK_KEYS = {
-    building = 'named',
-    landuse = 'named',
-    junction = 'named',
-    healthcare = 'named'
-}
-
-
-PRE_DELETE = tag_match{keys = {'note', 'note:*', 'source', 'source*', 'attribution',
-                               'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*',
-                               'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*',
-                               'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*',
-                               'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type',
-                               'ref:linz:*', 'is_in:postcode'},
-                       tags = {emergency = {'yes', 'no', 'fire_hydrant'},
-                               historic = {'yes', 'no'},
-                               military = {'yes', 'no'},
-                               natural = {'yes', 'no', 'coastline'},
-                               highway = {'no', 'turning_circle', 'mini_roundabout',
-                                          'noexit', 'crossing', 'give_way', 'stop'},
-                               railway = {'level_crossing', 'no', 'rail'},
-                               man_made = {'survey_point', 'cutline'},
-                               aerialway = {'pylon', 'no'},
-                               aeroway = {'no'},
-                               amenity = {'no'},
-                               club = {'no'},
-                               craft = {'no'},
-                               leisure = {'no'},
-                               office = {'no'},
-                               mountain_pass = {'no'},
-                               shop = {'no'},
-                               tourism = {'yes', 'no'},
-                               bridge = {'no'},
-                               tunnel = {'no'},
-                               waterway = {'riverbank'},
-                               building = {'no'},
-                               boundary = {'place'}}
-                      }
-
-POST_DELETE = tag_match{keys = {'tiger:*'}}
-
-PRE_EXTRAS = tag_match{keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*',
-                               'name:etymology', 'name:signed', 'name:botanical',
-                               'wikidata', '*:wikidata',
-                               'addr:street:name', 'addr:street:type'}
-                      }
-
-
-NAMES = tag_match{keys = {'name', 'name:*',
-                          'int_name', 'int_name:*',
-                          'nat_name', 'nat_name:*',
-                          'reg_name', 'reg_name:*',
-                          'loc_name', 'loc_name:*',
-                          'old_name', 'old_name:*',
-                          'alt_name', 'alt_name:*', 'alt_name_*',
-                          'official_name', 'official_name:*',
-                          'place_name', 'place_name:*',
-                          'short_name', 'short_name:*', 'brand'}}
-
-REFS = tag_match{keys = {'ref', 'int_ref', 'nat_ref', 'reg_ref', 'loc_ref', 'old_ref',
-                         'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}}
-
-POSTCODES = tag_match{keys = {'postal_code', 'postcode', 'addr:postcode',
-                              'tiger:zip_left', 'tiger:zip_right'}}
-
-COUNTRY_TAGS = tag_match{keys = {'country_code', 'ISO3166-1',
-                                 'addr:country_code', 'is_in:country_code',
-                                 'addr:country', 'is_in:country'}}
-
-HOUSENAME_TAGS = tag_match{keys = {'addr:housename'}}
-
-HOUSENUMBER_TAGS = tag_match{keys = {'addr:housenumber', 'addr:conscriptionnumber',
-                                     'addr:streetnumber'}}
-
-INTERPOLATION_TAGS = tag_match{keys = {'addr:interpolation'}}
-
-ADDRESS_TAGS = tag_match{keys = {'addr:*', 'is_in:*', 'tiger:county'}}
-
-SAVE_EXTRA_MAINS = true
-

test/bdd/api/lookup/simple.feature

@@ -31,11 +31,3 @@ Feature: Places by osm_type and osm_id Tests
       | jsonv2 |
       | geojson |
       | xml |
-
-
-    Scenario: Lookup of a linked place
-        When sending geocodejson lookup query for N1932181216
-        Then exactly 1 result is returned
-        And results contain
-          | name  |
-          | Vaduz |

test/bdd/api/search/params.feature

@@ -68,15 +68,14 @@ Feature: Search queries
           | 0 |
         Then there are duplicates
 
-    @fail-legacy
     Scenario: Search with bounded viewbox in right area
-        When sending json search query "post" with address
+        When sending json search query "bar" with address
           | bounded | viewbox |
           | 1       |  9,47,10,48 |
         Then result addresses contain
           | ID | town |
           | 0  | Vaduz |
-        When sending json search query "post" with address
+        When sending json search query "bar" with address
           | bounded | viewbox |
           | 1       |  9.49712,47.17122,9.52605,47.16242 |
         Then result addresses contain
@@ -119,18 +118,18 @@ Feature: Search queries
         Then result has centroid in 9.49712,47.16242,9.52605,47.17122
 
     Scenario: Prefer results within viewbox
+        When sending json search query "Gässle" with address
+          | accept-language |
+          | en |
+        Then result addresses contain
+          | ID | town |
+          | 0  | Balzers |
         When sending json search query "Gässle" with address
           | accept-language | viewbox |
           | en              | 9.52413,47.10759,9.53140,47.10539 |
         Then result addresses contain
           | ID | village |
           | 0  | Triesen |
-        When sending json search query "Gässle" with address
-          | accept-language | viewbox |
-          | en              | 9.45949,47.08421,9.54094,47.05466 |
-        Then result addresses contain
-          | ID | town |
-          | 0  | Balzers |
 
     Scenario: viewboxes cannot be points
         When sending json search query "foo"
@@ -172,12 +171,10 @@ Feature: Search queries
 
     Scenario: Limit number of search results
         When sending json search query "landstr"
-          | dedupe |
-          | 0      |
         Then more than 4 results are returned
         When sending json search query "landstr"
-          | limit | dedupe |
+          | limit |
-          | 4     | 0      |
+          | 4 |
         Then exactly 4 results are returned
 
     Scenario: Limit parameter must be a number

test/bdd/db/import/parenting.feature

@@ -437,29 +437,6 @@ Feature: Parenting of objects
           | object | parent_place_id |
           | N9     | R14             |
 
-
-    Scenario: Choose closest street in associatedStreet relation
-        Given the grid
-         | 1  |   |    |  | 3  |
-         | 10 |   | 11 |  | 12 |
-        And the places
-         | osm | class | type  | housenr | geometry |
-         | N1  | place | house | 1       | 1        |
-         | N3  | place | house | 3       | 3        |
-        And the named places
-         | osm  | class    | type        | geometry |
-         | W100 | highway  | residential | 10,11    |
-         | W101 | highway  | residential | 11,12    |
-        And the relations
-         | id | members                                            | tags+type |
-         | 1  | N1:house,N3:house,W100:street,W101:street | associatedStreet |
-        When importing
-        Then placex contains
-         | object | parent_place_id |
-         | N1     | W100            |
-         | N3     | W101            |
-
-
     Scenario: POIs in building inherit address
         Given the grid
          | 10 |  |   |   |   |   | 11 |

test/bdd/environment.py

@@ -27,7 +27,6 @@ userconfig = {
     'API_TEST_FILE'  : (TEST_BASE_DIR / 'testdb' / 'apidb-test-data.pbf').resolve(),
     'SERVER_MODULE_PATH' : None,
     'TOKENIZER' : None, # Test with a custom tokenizer
-    'STYLE' : 'extratags',
     'PHPCOV' : False, # set to output directory to enable code coverage
 }
 

test/bdd/osm2pgsql/import/tags.feature

@@ -1,195 +0,0 @@
-@DB
-Feature: Tag evaluation
-    Tests if tags are correctly imported into the place table
-
-    Scenario: Main tags as fallback
-        When loading osm data
-            """
-            n100 Tjunction=yes,highway=bus_stop
-            n101 Tjunction=yes,name=Bar
-            n200 Tbuilding=yes,amenity=cafe
-            n201 Tbuilding=yes,name=Intersting
-            n202 Tbuilding=yes
-            """
-        Then place contains exactly
-            | object | class    | type     |
-            | N100   | highway  | bus_stop |
-            | N101   | junction | yes      |
-            | N200   | amenity  | cafe     |
-            | N201   | building | yes      |
-
-
-    Scenario: Name and reg tags
-        When loading osm data
-            """
-            n2001 Thighway=road,name=Foo,alt_name:de=Bar,ref=45
-            n2002 Thighway=road,name:prefix=Pre,name:suffix=Post,ref:de=55
-            n2003 Thighway=yes,name:%20%de=Foo,name=real1
-            n2004 Thighway=yes,name:%a%de=Foo,name=real2
-            n2005 Thighway=yes,name:%9%de=Foo,name:\\=real3
-            n2006 Thighway=yes,name:%9%de=Foo,name=rea\l3
-            """
-        Then place contains exactly
-            | object | class   | type | name |
-            | N2001  | highway | road | 'name': 'Foo', 'alt_name:de': 'Bar', 'ref': '45' |
-            | N2002  | highway | road | - |
-            | N2003  | highway | yes  | 'name: de': 'Foo', 'name': 'real1' |
-            | N2004  | highway | yes  | 'name:\nde': 'Foo', 'name': 'real2' |
-            | N2005  | highway | yes  | 'name:\tde': 'Foo', 'name:\\\\': 'real3' |
-            | N2006  | highway | yes  | 'name:\tde': 'Foo', 'name': 'rea\\l3' |
-
-        And place contains
-            | object | extratags |
-            | N2002  | 'name:prefix': 'Pre', 'name:suffix': 'Post', 'ref:de': '55' |
-
-
-    Scenario: Name when using with_name flag
-        When loading osm data
-            """
-            n3001 Tbridge=yes,bridge:name=GoldenGate
-            n3002 Tbridge=yes,bridge:name:en=Rainbow
-            """
-        Then place contains exactly
-            | object | class   | type | name                 |
-            | N3001  | bridge  | yes  | 'name': 'GoldenGate' |
-            | N3002  | bridge  | yes  | 'name:en': 'Rainbow' |
-
-
-    Scenario: Address tags
-        When loading osm data
-            """
-            n4001 Taddr:housenumber=34,addr:city=Esmarald,addr:county=Land
-            n4002 Taddr:streetnumber=10,is_in:city=Rootoo,is_in=Gold
-            """
-        Then place contains exactly
-            | object | class | address             |
-            | N4001  | place | 'housenumber': '34', 'city': 'Esmarald', 'county': 'Land' |
-            | N4002  | place | 'streetnumber': '10', 'city': 'Rootoo' |
-
-
-    Scenario: Country codes
-        When loading osm data
-            """
-            n5001 Tshop=yes,country_code=DE
-            n5002 Tshop=yes,country_code=toolong
-            n5003 Tshop=yes,country_code=x
-            n5004 Tshop=yes,addr:country=us
-            n5005 Tshop=yes,country=be
-            n5006 Tshop=yes,addr:country=France
-            """
-        Then place contains exactly
-            | object | class | address         |
-            | N5001  | shop  | 'country': 'DE' |
-            | N5002  | shop  | - |
-            | N5003  | shop  | - |
-            | N5004  | shop  | 'country': 'us' |
-            | N5005  | shop  | - |
-            | N5006  | shop  | - |
-
-
-    Scenario: Postcodes
-        When loading osm data
-            """
-            n6001 Tshop=bank,addr:postcode=12345
-            n6002 Tshop=bank,tiger:zip_left=34343
-            n6003 Tshop=bank,is_in:postcode=9009
-            """
-        Then place contains exactly
-            | object | class | address             |
-            | N6001  | shop  | 'postcode': '12345' |
-            | N6002  | shop  | 'postcode': '34343' |
-            | N6003  | shop  | -                   |
-
-
-    Scenario: Main with extra
-        When loading osm data
-            """
-            n7001 Thighway=primary,bridge=yes,name=1
-            n7002 Thighway=primary,bridge=yes,bridge:name=1
-            """
-        Then place contains exactly
-            | object        | class   | type    | name        | extratags+bridge:name |
-            | N7001         | highway | primary | 'name': '1' | -                     |
-            | N7002:highway | highway | primary | -           | 1                     |
-            | N7002:bridge  | bridge  | yes     | 'name': '1' | 1                     |
-
-
-    Scenario: Global fallback and skipping
-        When loading osm data
-            """
-            n8001 Tshop=shoes,note:de=Nein,xx=yy
-            n8002 Tshop=shoes,building=no,ele=234
-            n8003 Tshop=shoes,name:source=survey
-            """
-        Then place contains exactly
-            | object | class | extratags    |
-            | N8001  | shop  | 'xx': 'yy'   |
-            | N8002  | shop  | 'ele': '234' |
-            | N8003  | shop  | -            |
-
-
-    Scenario: Admin levels
-        When loading osm data
-            """
-            n9001 Tplace=city
-            n9002 Tplace=city,admin_level=16
-            n9003 Tplace=city,admin_level=x
-            n9004 Tplace=city,admin_level=1
-            n9005 Tplace=city,admin_level=0
-            n9006 Tplace=city,admin_level=2.5
-            """
-        Then place contains exactly
-            | object | class | admin_level |
-            | N9001  | place | 15          |
-            | N9002  | place | 15          |
-            | N9003  | place | 15          |
-            | N9004  | place | 1           |
-            | N9005  | place | 15          |
-            | N9006  | place | 15          |
-
-
-    Scenario: Administrative boundaries with place tags
-        When loading osm data
-            """
-            n10001 Tboundary=administrative,place=city,name=A
-            n10002 Tboundary=natural,place=city,name=B
-            n10003 Tboundary=administrative,place=island,name=C
-            """
-        Then place contains
-            | object          | class    | type           | extratags       |
-            | N10001          | boundary | administrative | 'place': 'city' |
-        And place contains
-            | object          | class    | type           |
-            | N10002:boundary | boundary | natural        |
-            | N10002:place    | place    | city           |
-            | N10003:boundary | boundary | administrative |
-            | N10003:place    | place    | island         |
-
-
-    Scenario: Building fallbacks
-        When loading osm data
-            """
-            n12001 Ttourism=hotel,building=yes
-            n12002 Tbuilding=house
-            n12003 Tbuilding=shed,addr:housenumber=1
-            n12004 Tbuilding=yes,name=Das-Haus
-            n12005 Tbuilding=yes,addr:postcode=12345
-            """
-        Then place contains exactly
-            | object | class    | type     |
-            | N12001 | tourism  | hotel    |
-            | N12003 | building | shed     |
-            | N12004 | building | yes      |
-            | N12005 | place    | postcode |
-
-
-    Scenario: Address interpolations
-        When loading osm data
-            """
-            n13001 Taddr:interpolation=odd
-            n13002 Taddr:interpolation=even,place=city
-            """
-        Then place contains exactly
-            | object | class | type   | address                 |
-            | N13001 | place | houses | 'interpolation': 'odd'  |
-            | N13002 | place | houses | 'interpolation': 'even' |

test/bdd/osm2pgsql/update/interpolations.feature

@@ -1,133 +0,0 @@
-@DB
-Feature: Updates of address interpolation objects
-    Test that changes to address interpolation objects are correctly
-    propagated.
-
-    Background:
-        Given the grid
-            | 1 | 2 |
-
-
-    Scenario: Adding a new interpolation
-        When loading osm data
-            """
-            n1 Taddr:housenumber=3
-            n2 Taddr:housenumber=17
-            """
-        Then place contains
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-
-        When updating osm data
-            """
-            w99 Taddr:interpolation=odd Nn1,n2
-            """
-        Then place contains
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-            | W99:place | houses |
-        When indexing
-        Then placex contains exactly
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-        Then location_property_osmline contains exactly
-            | object |
-            | 99:5   |
-
-
-    Scenario: Delete an existing interpolation
-        When loading osm data
-            """
-            n1 Taddr:housenumber=2
-            n2 Taddr:housenumber=7
-            w99 Taddr:interpolation=odd Nn1,n2
-            """
-        Then place contains
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-            | W99:place | houses |
-
-        When updating osm data
-            """
-            w99 v2 dD
-            """
-        Then place contains
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-        When indexing
-        Then placex contains exactly
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-        Then location_property_osmline contains exactly
-            | object | indexed_status |
-
-
-    Scenario: Changing an object to an interpolation
-        When loading osm data
-            """
-            n1 Taddr:housenumber=3
-            n2 Taddr:housenumber=17
-            w99 Thighway=residential Nn1,n2
-            """
-        Then place contains
-            | object      | type   |
-            | N1:place    | house  |
-            | N2:place    | house  |
-            | W99:highway | residential  |
-
-        When updating osm data
-            """
-            w99 Taddr:interpolation=odd Nn1,n2
-            """
-        Then place contains
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-            | W99:place | houses |
-        When indexing
-        Then placex contains exactly
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-        And location_property_osmline contains exactly
-            | object |
-            | 99:5   |
-
-
-    Scenario: Changing an interpolation to something else
-        When loading osm data
-            """
-            n1 Taddr:housenumber=3
-            n2 Taddr:housenumber=17
-            w99 Taddr:interpolation=odd Nn1,n2
-            """
-        Then place contains
-            | object      | type   |
-            | N1:place    | house  |
-            | N2:place    | house  |
-            | W99:place | houses |
-
-        When updating osm data
-            """
-            w99 Thighway=residential Nn1,n2
-            """
-        Then place contains
-            | object      | type   |
-            | N1:place    | house  |
-            | N2:place    | house  |
-            | W99:highway | residential  |
-        When indexing
-        Then placex contains exactly
-            | object      | type   |
-            | N1:place    | house  |
-            | N2:place    | house  |
-            | W99:highway | residential  |
-        And location_property_osmline contains exactly
-            | object |
-

test/bdd/osm2pgsql/update/postcodes.feature

@@ -1,163 +0,0 @@
-@DB
-Feature: Update of postcode only objects
-    Tests that changes to objects containing only a postcode are
-    propagated correctly.
-
-
-    Scenario: Adding a postcode-only node
-        When loading osm data
-            """
-            """
-        Then place contains exactly
-            | object |
-
-        When updating osm data
-            """
-            n34 Tpostcode=4456
-            """
-        Then place contains exactly
-            | object    | type     |
-            | N34:place | postcode |
-        When indexing
-        Then placex contains exactly
-            | object |
-
-
-    Scenario: Deleting a postcode-only node
-        When loading osm data
-            """
-            n34 Tpostcode=4456
-            """
-        Then place contains exactly
-            | object    | type     |
-            | N34:place | postcode |
-
-        When updating osm data
-            """
-            n34 v2 dD
-            """
-        Then place contains exactly
-            | object |
-        When indexing
-        Then placex contains exactly
-            | object |
-
-
-    Scenario Outline: Converting a regular object into a postcode-only node
-        When loading osm data
-            """
-            n34 T<class>=<type>
-            """
-        Then place contains exactly
-            | object      | type   |
-            | N34:<class> | <type> |
-
-        When updating osm data
-            """
-            n34 Tpostcode=4456
-            """
-        Then place contains exactly
-            | object    | type     |
-            | N34:place | postcode |
-        When indexing
-        Then placex contains exactly
-            | object |
-
-        Examples:
-            | class   | type       |
-            | amenity | restaurant |
-            | place   | hamlet     |
-
-
-    Scenario Outline: Converting a postcode-only node into a regular object
-        When loading osm data
-            """
-            n34 Tpostcode=4456
-            """
-        Then place contains exactly
-            | object    | type     |
-            | N34:place | postcode |
-
-        When updating osm data
-            """
-            n34 T<class>=<type>
-            """
-        Then place contains exactly
-            | object      | type   |
-            | N34:<class> | <type> |
-        When indexing
-        Then placex contains exactly
-            | object      | type   |
-            | N34:<class> | <type> |
-
-        Examples:
-            | class   | type       |
-            | amenity | restaurant |
-            | place   | hamlet     |
-
-
-    Scenario: Converting na interpolation into a postcode-only node
-        Given the grid
-            | 1 | 2 |
-        When loading osm data
-            """
-            n1 Taddr:housenumber=3
-            n2 Taddr:housenumber=17
-            w34 Taddr:interpolation=odd Nn1,n2
-            """
-        Then place contains exactly
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-            | W34:place | houses |
-
-        When updating osm data
-            """
-            w34 Tpostcode=4456 Nn1,n2
-            """
-        Then place contains exactly
-            | object    | type     |
-            | N1:place  | house    |
-            | N2:place  | house    |
-            | W34:place | postcode |
-        When indexing
-        Then location_property_osmline contains exactly
-            | object |
-        And placex contains exactly
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-
-
-    Scenario: Converting a postcode-only node into an interpolation
-        Given the grid
-            | 1 | 2 |
-        When loading osm data
-            """
-            n1 Taddr:housenumber=3
-            n2 Taddr:housenumber=17
-            w34 Tpostcode=4456 Nn1,n2
-            """
-        Then place contains exactly
-            | object    | type     |
-            | N1:place  | house    |
-            | N2:place  | house    |
-            | W34:place | postcode |
-
-        When updating osm data
-            """
-            w34 Taddr:interpolation=odd Nn1,n2
-            """
-        Then place contains exactly
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |
-            | W34:place | houses |
-        When indexing
-        Then location_property_osmline contains exactly
-            | object |
-            | 34:5   |
-        And placex contains exactly
-            | object    | type   |
-            | N1:place  | house  |
-            | N2:place  | house  |

test/bdd/osm2pgsql/update/simple.feature

@@ -2,48 +2,60 @@
 Feature: Update of simple objects by osm2pgsql
     Testing basic update functions of osm2pgsql.
 
-    Scenario: Adding a new object
+    Scenario: Import object with two main tags
         When loading osm data
           """
-          n1 Tplace=town,name=Middletown
+          n1 Ttourism=hotel,amenity=restaurant,name=foo
+          n2 Tplace=locality,name=spotty
           """
-        Then place contains exactly
+        Then place contains
           | object     | type       | name+name |
-          | N1:place | town | Middletown |
+          | N1:tourism | hotel      | foo |
-
+          | N1:amenity | restaurant | foo |
-       When updating osm data
+          | N2:place   | locality   | spotty |
-         """
-         n2 Tamenity=hotel,name=Posthotel
-         """
-        Then place contains exactly
-          | object     | type  | name+name  |
-          | N1:place   | town  | Middletown |
-          | N2:amenity | hotel | Posthotel  |
-        And placex contains exactly
-          | object     | type  | name+name  | indexed_status |
-          | N1:place   | town  | Middletown | 0              |
-          | N2:amenity | hotel | Posthotel  | 1              |
-
-
-    Scenario: Deleting an existing object
-        When loading osm data
-          """
-          n1 Tplace=town,name=Middletown
-          n2 Tamenity=hotel,name=Posthotel
-          """
-        Then place contains exactly
-          | object     | type  | name+name  |
-          | N1:place   | town  | Middletown |
-          | N2:amenity | hotel | Posthotel  |
-
         When updating osm data
           """
+          n1 dV Ttourism=hotel,name=foo
           n2 dD
           """
-        Then place contains exactly
+        Then place has no entry for N1:amenity
-          | object     | type  | name+name  |
+        And place has no entry for N2
-          | N1:place   | town  | Middletown |
+        And place contains
-        And placex contains exactly
+          | object     | class   | type       | name |
-          | object     | type  | name+name  | indexed_status |
+          | N1:tourism | tourism | hotel      | 'name' : 'foo' |
-          | N1:place   | town  | Middletown | 0              |
+
-          | N2:amenity | hotel | Posthotel  | 100            |
+    Scenario: Downgrading a highway to one that is dropped without name
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=residential Nn100,n101
+          """
+        Then place contains
+          | object     |
+          | W1:highway |
+        When updating osm data
+          """
+          w1 Thighway=service Nn100,n101
+          """
+        Then place has no entry for W1
+
+    Scenario: Downgrading a highway when a second tag is present
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=residential,tourism=hotel Nn100,n101
+          """
+        Then place contains
+          | object     |
+          | W1:highway |
+          | W1:tourism |
+        When updating osm data
+          """
+          w1 Thighway=service,tourism=hotel Nn100,n101
+          """
+        Then place has no entry for W1:highway
+        And place contains
+          | object     |
+          | W1:tourism |

test/bdd/osm2pgsql/update/tags.feature

@@ -1,490 +0,0 @@
-@DB
-Feature: Tag evaluation
-    Tests if tags are correctly updated in the place table
-
-    Background:
-        Given the grid
-            | 1  | 2  | 3 |
-            | 10 | 11 |   |
-            | 45 | 46 |   |
-
-    Scenario: Main tag deleted
-        When loading osm data
-            """
-            n1 Tamenity=restaurant
-            n2 Thighway=bus_stop,railway=stop,name=X
-            n3 Tamenity=prison
-            """
-        Then place contains exactly
-            | object     | class   | type       |
-            | N1         | amenity | restaurant |
-            | N2:highway | highway | bus_stop   |
-            | N2:railway | railway | stop       |
-            | N3         | amenity | prison     |
-
-        When updating osm data
-            """
-            n1 Tnot_a=restaurant
-            n2 Thighway=bus_stop,name=X
-            """
-        Then place contains exactly
-            | object     | class   | type       |
-            | N2:highway | highway | bus_stop   |
-            | N3         | amenity | prison     |
-        And placex contains
-            | object     | indexed_status |
-            | N3:amenity | 0              |
-        When indexing
-        Then placex contains exactly
-            | object     | type     | name        |
-            | N2:highway | bus_stop | 'name': 'X' |
-            | N3:amenity | prison   | -           |
-
-
-    Scenario: Main tag added
-        When loading osm data
-            """
-            n1 Tatity=restaurant
-            n2 Thighway=bus_stop,name=X
-            """
-        Then place contains exactly
-            | object     | class   | type       |
-            | N2:highway | highway | bus_stop   |
-
-        When updating osm data
-            """
-            n1 Tamenity=restaurant
-            n2 Thighway=bus_stop,railway=stop,name=X
-            """
-        Then place contains exactly
-            | object     | class   | type       |
-            | N1         | amenity | restaurant |
-            | N2:highway | highway | bus_stop   |
-            | N2:railway | railway | stop       |
-        When indexing
-        Then placex contains exactly
-            | object     | type       | name        |
-            | N1:amenity | restaurant | -           |
-            | N2:highway | bus_stop   | 'name': 'X' |
-            | N2:railway | stop       | 'name': 'X' |
-
-
-    Scenario: Main tag modified
-        When loading osm data
-            """
-            n10 Thighway=footway,name=X
-            n11 Tamenity=atm
-            """
-        Then place contains exactly
-            | object | class   | type    |
-            | N10    | highway | footway |
-            | N11    | amenity | atm     |
-
-        When updating osm data
-            """
-            n10 Thighway=path,name=X
-            n11 Thighway=primary
-            """
-        Then place contains exactly
-            | object | class   | type    |
-            | N10    | highway | path    |
-            | N11    | highway | primary |
-        When indexing
-        Then placex contains exactly
-            | object      | type       | name        |
-            | N10:highway | path       | 'name': 'X' |
-            | N11:highway | primary    | -           |
-
-
-    Scenario: Main tags with name, name added
-        When loading osm data
-            """
-            n45 Tlanduse=cemetry
-            n46 Tbuilding=yes
-            """
-        Then place contains exactly
-            | object | class   | type    |
-
-        When updating osm data
-            """
-            n45 Tlanduse=cemetry,name=TODO
-            n46 Tbuilding=yes,addr:housenumber=1
-            """
-        Then place contains exactly
-            | object | class   | type    |
-            | N45    | landuse | cemetry |
-            | N46    | building| yes     |
-        When indexing
-        Then placex contains exactly
-            | object      | type       | name           | address            |
-            | N45:landuse | cemetry    | 'name': 'TODO' | -                  |
-            | N46:building| yes        | -              | 'housenumber': '1' |
-
-
-    Scenario: Main tags with name, name removed
-        When loading osm data
-            """
-            n45 Tlanduse=cemetry,name=TODO
-            n46 Tbuilding=yes,addr:housenumber=1
-            """
-        Then place contains exactly
-            | object | class   | type    |
-            | N45    | landuse | cemetry |
-            | N46    | building| yes     |
-
-        When updating osm data
-            """
-            n45 Tlanduse=cemetry
-            n46 Tbuilding=yes
-            """
-        Then place contains exactly
-            | object | class   | type    |
-        When indexing
-        Then placex contains exactly
-            | object      |
-
-    Scenario: Main tags with name, name modified
-        When loading osm data
-            """
-            n45 Tlanduse=cemetry,name=TODO
-            n46 Tbuilding=yes,addr:housenumber=1
-            """
-        Then place contains exactly
-            | object | class   | type    | name            | address           |
-            | N45    | landuse | cemetry | 'name' : 'TODO' | -                 |
-            | N46    | building| yes     | -               | 'housenumber': '1'|
-
-        When updating osm data
-            """
-            n45 Tlanduse=cemetry,name=DONE
-            n46 Tbuilding=yes,addr:housenumber=10
-            """
-        Then place contains exactly
-            | object | class   | type    | name            | address            |
-            | N45    | landuse | cemetry | 'name' : 'DONE' | -                  |
-            | N46    | building| yes     | -               | 'housenumber': '10'|
-        When indexing
-        Then placex contains exactly
-            | object | class   | type    | name            | address            |
-            | N45    | landuse | cemetry | 'name' : 'DONE' | -                  |
-            | N46    | building| yes     | -               | 'housenumber': '10'|
-
-
-    Scenario: Main tag added to address only node
-        When loading osm data
-            """
-            n1 Taddr:housenumber=345
-            """
-        Then place contains exactly
-            | object | class | type  | address |
-            | N1     | place | house | 'housenumber': '345'|
-
-        When updating osm data
-            """
-            n1 Taddr:housenumber=345,building=yes
-            """
-        Then place contains exactly
-            | object | class    | type  | address |
-            | N1     | building | yes   | 'housenumber': '345'|
-        When indexing
-        Then placex contains exactly
-            | object | class    | type  | address |
-            | N1     | building | yes   | 'housenumber': '345'|
-
-
-    Scenario: Main tag removed from address only node
-        When loading osm data
-            """
-            n1 Taddr:housenumber=345,building=yes
-            """
-        Then place contains exactly
-            | object | class    | type  | address |
-            | N1     | building | yes   | 'housenumber': '345'|
-
-        When updating osm data
-            """
-            n1 Taddr:housenumber=345
-            """
-        Then place contains exactly
-            | object | class | type  | address |
-            | N1     | place | house | 'housenumber': '345'|
-        When indexing
-        Then placex contains exactly
-            | object | class | type  | address |
-            | N1     | place | house | 'housenumber': '345'|
-
-
-    Scenario: Main tags with name key, adding key name
-        When loading osm data
-            """
-            n2 Tbridge=yes
-            """
-        Then place contains exactly
-            | object | class    | type  |
-
-        When updating osm data
-            """
-            n2 Tbridge=yes,bridge:name=high
-            """
-        Then place contains exactly
-            | object | class    | type  | name           |
-            | N2     | bridge   | yes   | 'name': 'high' |
-        When indexing
-        Then placex contains exactly
-            | object | class    | type  | name           |
-            | N2     | bridge   | yes   | 'name': 'high' |
-
-
-    Scenario: Main tags with name key, deleting key name
-        When loading osm data
-            """
-            n2 Tbridge=yes,bridge:name=high
-            """
-        Then place contains exactly
-            | object | class    | type  | name           |
-            | N2     | bridge   | yes   | 'name': 'high' |
-
-        When updating osm data
-            """
-            n2 Tbridge=yes
-            """
-        Then place contains exactly
-            | object |
-        When indexing
-        Then placex contains exactly
-            | object |
-
-
-    Scenario: Main tags with name key, changing key name
-        When loading osm data
-            """
-            n2 Tbridge=yes,bridge:name=high
-            """
-        Then place contains exactly
-            | object | class    | type  | name           |
-            | N2     | bridge   | yes   | 'name': 'high' |
-
-        When updating osm data
-            """
-            n2 Tbridge=yes,bridge:name:en=high
-            """
-        Then place contains exactly
-            | object | class  | type | name              |
-            | N2     | bridge | yes  | 'name:en': 'high' |
-        When indexing
-        Then placex contains exactly
-            | object | class  | type | name              |
-            | N2     | bridge | yes  | 'name:en': 'high' |
-
-
-    Scenario: Downgrading a highway to one that is dropped without name
-        When loading osm data
-          """
-          n100 x0 y0
-          n101 x0.0001 y0.0001
-          w1 Thighway=residential Nn100,n101
-          """
-        Then place contains exactly
-          | object     |
-          | W1:highway |
-
-        When updating osm data
-          """
-          w1 Thighway=service Nn100,n101
-          """
-        Then place contains exactly
-          | object     |
-        When indexing
-        Then placex contains exactly
-            | object |
-
-
-    Scenario: Upgrading a highway to one that is not dropped without name
-        When loading osm data
-          """
-          n100 x0 y0
-          n101 x0.0001 y0.0001
-          w1 Thighway=service Nn100,n101
-          """
-        Then place contains exactly
-          | object     |
-
-        When updating osm data
-          """
-          w1 Thighway=unclassified Nn100,n101
-          """
-        Then place contains exactly
-          | object     |
-          | W1:highway |
-        When indexing
-        Then placex contains exactly
-          | object     |
-          | W1:highway |
-
-
-    Scenario: Downgrading a highway when a second tag is present
-        When loading osm data
-          """
-          n100 x0 y0
-          n101 x0.0001 y0.0001
-          w1 Thighway=residential,tourism=hotel Nn100,n101
-          """
-        Then place contains exactly
-          | object     | type        |
-          | W1:highway | residential |
-          | W1:tourism | hotel       |
-
-        When updating osm data
-          """
-          w1 Thighway=service,tourism=hotel Nn100,n101
-          """
-        Then place contains exactly
-          | object     | type  |
-          | W1:tourism | hotel |
-        When indexing
-        Then placex contains exactly
-          | object     | type  |
-          | W1:tourism | hotel |
-
-
-    Scenario: Upgrading a highway when a second tag is present
-        When loading osm data
-          """
-          n100 x0 y0
-          n101 x0.0001 y0.0001
-          w1 Thighway=service,tourism=hotel Nn100,n101
-          """
-        Then place contains exactly
-          | object     | type  |
-          | W1:tourism | hotel |
-
-        When updating osm data
-          """
-          w1 Thighway=residential,tourism=hotel Nn100,n101
-          """
-        Then place contains exactly
-          | object     | type        |
-          | W1:highway | residential |
-          | W1:tourism | hotel       |
-        When indexing
-        Then placex contains exactly
-          | object     | type        |
-          | W1:highway | residential |
-          | W1:tourism | hotel       |
-
-
-    Scenario: Replay on administrative boundary
-        When loading osm data
-          """
-          n10 x34.0 y-4.23
-          n11 x34.1 y-4.23
-          n12 x34.2 y-4.13
-          w10 Tboundary=administrative,waterway=river,name=Border,admin_level=2 Nn12,n11,n10
-          """
-        Then place contains exactly
-          | object       | type           | admin_level | name             |
-          | W10:waterway | river          | 2           | 'name': 'Border' |
-          | W10:boundary | administrative | 2           | 'name': 'Border' |
-
-        When updating osm data
-          """
-          w10 Tboundary=administrative,waterway=river,name=Border,admin_level=2 Nn12,n11,n10
-          """
-        Then place contains exactly
-          | object       | type           | admin_level | name             |
-          | W10:waterway | river          | 2           | 'name': 'Border' |
-          | W10:boundary | administrative | 2           | 'name': 'Border' |
-        When indexing
-        Then placex contains exactly
-          | object       | type           | admin_level | name             |
-          | W10:waterway | river          | 2           | 'name': 'Border' |
-
-
-    Scenario: Change admin_level on administrative boundary
-        Given the grid
-          | 10 | 11 |
-          | 13 | 12 |
-        When loading osm data
-          """
-          n10
-          n11
-          n12
-          n13
-          w10 Nn10,n11,n12,n13,n10
-          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=2 Mw10@
-          """
-        Then place contains exactly
-          | object       | admin_level |
-          | R10:boundary | 2           |
-
-        When updating osm data
-          """
-          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
-          """
-        Then place contains exactly
-          | object       | type           | admin_level |
-          | R10:boundary | administrative | 4           |
-        When indexing
-        Then placex contains exactly
-          | object       | type           | admin_level |
-          | R10:boundary | administrative | 4           |
-
-
-    Scenario: Change boundary to administrative
-        Given the grid
-          | 10 | 11 |
-          | 13 | 12 |
-        When loading osm data
-          """
-          n10
-          n11
-          n12
-          n13
-          w10 Nn10,n11,n12,n13,n10
-          r10 Ttype=multipolygon,boundary=informal,name=Border,admin_level=4 Mw10@
-          """
-        Then place contains exactly
-          | object       | type     | admin_level |
-          | R10:boundary | informal | 4           |
-
-        When updating osm data
-          """
-          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
-          """
-        Then place contains exactly
-          | object       | type           | admin_level |
-          | R10:boundary | administrative | 4           |
-        When indexing
-        Then placex contains exactly
-          | object       | type           | admin_level |
-          | R10:boundary | administrative | 4           |
-
-
-    Scenario: Change boundary away from administrative
-        Given the grid
-          | 10 | 11 |
-          | 13 | 12 |
-        When loading osm data
-          """
-          n10
-          n11
-          n12
-          n13
-          w10 Nn10,n11,n12,n13,n10
-          r10 Ttype=multipolygon,boundary=administrative,name=Border,admin_level=4 Mw10@
-          """
-        Then place contains exactly
-          | object       | type           | admin_level |
-          | R10:boundary | administrative | 4           |
-
-        When updating osm data
-          """
-          r10 Ttype=multipolygon,boundary=informal,name=Border,admin_level=4 Mw10@
-          """
-        Then place contains exactly
-          | object       | type     | admin_level |
-          | R10:boundary | informal | 4           |
-        When indexing
-        Then placex contains exactly
-          | object       | type     | admin_level |
-          | R10:boundary | informal | 4           |

test/bdd/steps/nominatim_environment.py

@@ -36,7 +36,6 @@ class NominatimEnvironment:
         self.api_test_db = config['API_TEST_DB']
         self.api_test_file = config['API_TEST_FILE']
         self.tokenizer = config['TOKENIZER']
-        self.import_style = config['STYLE']
         self.server_module_path = config['SERVER_MODULE_PATH']
         self.reuse_template = not config['REMOVE_TEMPLATE']
         self.keep_scenario_db = config['KEEP_TEST_DB']
@@ -108,8 +107,6 @@ class NominatimEnvironment:
         self.test_env['NOMINATIM_NOMINATIM_TOOL'] = str((self.build_dir / 'nominatim').resolve())
         if self.tokenizer is not None:
             self.test_env['NOMINATIM_TOKENIZER'] = self.tokenizer
-        if self.import_style is not None:
-            self.test_env['NOMINATIM_IMPORT_STYLE'] = self.import_style
 
         if self.server_module_path:
             self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path
@@ -204,21 +201,19 @@ class NominatimEnvironment:
             self.api_db_done = True
 
             if not self._reuse_or_drop_db(self.api_test_db):
-                testdata = (Path(__file__) / '..' / '..' / '..' / 'testdb').resolve()
+                testdata = Path('__file__') / '..' / '..' / 'testdb'
-                self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata)
+                self.test_env['NOMINATIM_WIKIPEDIA_DATA_PATH'] = str(testdata.resolve())
-                simp_file = Path(self.website_dir.name) / 'secondary_importance.sql.gz'
-                simp_file.symlink_to(testdata / 'secondary_importance.sql.gz')
 
                 try:
                     self.run_nominatim('import', '--osm-file', str(self.api_test_file))
-                    self.run_nominatim('add-data', '--tiger-data', str(testdata / 'tiger'))
+                    self.run_nominatim('add-data', '--tiger-data', str((testdata / 'tiger').resolve()))
                     self.run_nominatim('freeze')
 
                     if self.tokenizer == 'legacy':
-                        phrase_file = str(testdata / 'specialphrases_testdb.sql')
+                        phrase_file = str((testdata / 'specialphrases_testdb.sql').resolve())
                         run_script(['psql', '-d', self.api_test_db, '-f', phrase_file])
                     else:
-                        csv_path = str(testdata / 'full_en_phrases_test.csv')
+                        csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
                         self.run_nominatim('special-phrases', '--import-from-csv', csv_path)
                 except:
                     self.db_drop_database(self.api_test_db)

test/bdd/steps/place_inserter.py

@@ -92,12 +92,6 @@ class PlaceColumn:
         else:
             self.columns[column] = {key: value}
 
-    def db_delete(self, cursor):
-        """ Issue a delete for the given OSM object.
-        """
-        cursor.execute('DELETE FROM place WHERE osm_type = %s and osm_id = %s',
-                       (self.columns['osm_type'] , self.columns['osm_id']))
-
     def db_insert(self, cursor):
         """ Insert the collected data into the database.
         """

test/bdd/steps/steps_db_ops.py

@@ -118,10 +118,7 @@ def update_place_table(context):
     context.nominatim.run_nominatim('refresh', '--functions')
     with context.db.cursor() as cur:
         for row in context.table:
-            col = PlaceColumn(context).add_row(row, False)
+            PlaceColumn(context).add_row(row, False).db_insert(cur)
-            col.db_delete(cur)
-            col.db_insert(cur)
-        cur.execute('SELECT flush_deleted_places()')
 
     context.nominatim.reindex_placex(context.db)
     check_database_integrity(context)
@@ -146,10 +143,8 @@ def delete_places(context, oids):
     """
     context.nominatim.run_nominatim('refresh', '--functions')
     with context.db.cursor() as cur:
-        cur.execute('TRUNCATE place_to_be_deleted')
         for oid in oids.split(','):
             NominatimID(oid).query_osm_id(cur, 'DELETE FROM place WHERE {}')
-        cur.execute('SELECT flush_deleted_places()')
 
     context.nominatim.reindex_placex(context.db)
 
@@ -190,10 +185,7 @@ def check_place_contents(context, table, exact):
 
         if exact:
             cur.execute('SELECT osm_type, osm_id, class from {}'.format(table))
-            actual = set([(r[0], r[1], r[2]) for r in cur])
+            assert expected_content == set([(r[0], r[1], r[2]) for r in cur])
-            assert expected_content == actual, \
-                   f"Missing entries: {expected_content - actual}\n" \
-                   f"Not expected in table: {actual - expected_content}"
 
 
 @then("(?P<table>placex|place) has no entry for (?P<oid>.*)")
@@ -380,49 +372,4 @@ def check_location_property_osmline(context, oid, neg):
 
         assert not todo, f"Unmatched lines in table: {list(context.table[i] for i in todo)}"
 
-@then("location_property_osmline contains(?P<exact> exactly)?")
-def check_place_contents(context, exact):
-    """ Check contents of the interpolation table. Each row represents a table row
-        and all data must match. Data not present in the expected table, may
-        be arbitry. The rows are identified via the 'object' column which must
-        have an identifier of the form '<osm id>[:<startnumber>]'. When multiple
-        rows match (for example because 'startnumber' was left out and there are
-        multiple entries for the given OSM object) then all must match. All
-        expected rows are expected to be present with at least one database row.
-        When 'exactly' is given, there must not be additional rows in the database.
-    """
-    with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
-        expected_content = set()
-        for row in context.table:
-            if ':' in row['object']:
-                nid, start = row['object'].split(':', 2)
-                start = int(start)
-            else:
-                nid, start = row['object'], None
-
-            query = """SELECT *, ST_AsText(linegeo) as geomtxt,
-                              ST_GeometryType(linegeo) as geometrytype
-                       FROM location_property_osmline WHERE osm_id=%s"""
-
-            if ':' in row['object']:
-                query += ' and startnumber = %s'
-                params = [int(val) for val in row['object'].split(':', 2)]
-            else:
-                params = (int(row['object']), )
-
-            cur.execute(query, params)
-            assert cur.rowcount > 0, "No rows found for " + row['object']
-
-            for res in cur:
-                if exact:
-                    expected_content.add((res['osm_id'], res['startnumber']))
-
-                DBRow(nid, res, context).assert_row(row, ['object'])
-
-        if exact:
-            cur.execute('SELECT osm_id, startnumber from location_property_osmline')
-            actual = set([(r[0], r[1]) for r in cur])
-            assert expected_content == actual, \
-                   f"Missing entries: {expected_content - actual}\n" \
-                   f"Not expected in table: {actual - expected_content}"
 

test/bdd/steps/steps_osm_data.py

@@ -10,7 +10,6 @@ import os
 from pathlib import Path
 
 from nominatim.tools.exec_utils import run_osm2pgsql
-from nominatim.tools.replication import run_osm2pgsql_updates
 
 from geometry_alias import ALIASES
 
@@ -18,8 +17,7 @@ def get_osm2pgsql_options(nominatim_env, fname, append):
     return dict(import_file=fname,
                 osm2pgsql=str(nominatim_env.build_dir / 'osm2pgsql' / 'osm2pgsql'),
                 osm2pgsql_cache=50,
-                osm2pgsql_style=str(nominatim_env.get_test_config().get_import_style_file()),
+                osm2pgsql_style=str(nominatim_env.src_dir / 'settings' / 'import-extratags.style'),
-                osm2pgsql_style_path=nominatim_env.get_test_config().config_dir,
                 threads=1,
                 dsn=nominatim_env.get_libpq_dsn(),
                 flatnode_file='',
@@ -119,15 +117,6 @@ def update_from_osm_file(context):
     # create an OSM file and import it
     fname = write_opl_file(context.text, context.osm)
     try:
-        run_osm2pgsql_updates(context.db,
+        run_osm2pgsql(get_osm2pgsql_options(context.nominatim, fname, append=True))
-                              get_osm2pgsql_options(context.nominatim, fname, append=True))
     finally:
         os.remove(fname)
-
-@when('indexing')
-def index_database(context):
-    """
-    Run the Nominatim indexing step. This will process data previously
-    loaded with 'updating osm data'
-    """
-    context.nominatim.run_nominatim('index')

test/php/Nominatim/DebugTest.php

@@ -37,14 +37,14 @@ class DebugTest extends \PHPUnit\Framework\TestCase
 <pre><b>Var1:</b>  <i>True</i></pre>
 <pre><b>Var2:</b>  <i>False</i></pre>
 <pre><b>Var3:</b>  0</pre>
-<pre><b>Var4:</b>  'String'</pre>
+<pre><b>Var4:</b>  &#039;String&#039;</pre>
-<pre><b>Var5:</b>  0 => 'one'
+<pre><b>Var5:</b>  0 => &#039;one&#039;
-       1 => 'two'
+       1 => &#039;two&#039;
-       2 => 'three'</pre>
+       2 => &#039;three&#039;</pre>
-<pre><b>Var6:</b>  'key' => 'value'
+<pre><b>Var6:</b>  &#039;key&#039; => &#039;value&#039;
-       'key2' => 'value2'</pre>
+       &#039;key2&#039; => &#039;value2&#039;</pre>
 <pre><b>Var7:</b>  me as string</pre>
-<pre><b>Var8:</b>  'value', 'value2'</pre>
+<pre><b>Var8:</b>  &#039;value&#039;, &#039;value2&#039;</pre>
 
 EOT
         );
@@ -64,10 +64,10 @@ EOT
     public function testDebugArray()
     {
         $this->expectOutputString(<<<EOT
-<pre><b>Arr0:</b>  'null'</pre>
+<pre><b>Arr0:</b>  &#039;null&#039;</pre>
-<pre><b>Arr1:</b>  'key1' => 'val1'
+<pre><b>Arr1:</b>  &#039;key1&#039; => &#039;val1&#039;
-       'key2' => 'val2'
+       &#039;key2&#039; => &#039;val2&#039;
-       'key3' => 'val3'</pre>
+       &#039;key3&#039; => &#039;val3&#039;</pre>
 
 EOT
         );
@@ -93,12 +93,12 @@ EOT
     <th><small>1</small></th>
   </tr>
   <tr>
-    <td><pre>'one'</pre></td>
+    <td><pre>&#039;one&#039;</pre></td>
-    <td><pre>'two'</pre></td>
+    <td><pre>&#039;two&#039;</pre></td>
   </tr>
   <tr>
-    <td><pre>'three'</pre></td>
+    <td><pre>&#039;three&#039;</pre></td>
-    <td><pre>'four'</pre></td>
+    <td><pre>&#039;four&#039;</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -109,9 +109,9 @@ EOT
     <th><small>key3</small></th>
   </tr>
   <tr>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
 </table>
 
@@ -147,18 +147,18 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'one'</pre></td>
+    <td><pre>&#039;one&#039;</pre></td>
-    <td><pre>'two'</pre></td>
+    <td><pre>&#039;two&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group2</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
   </tr>
 </table>
 <b>Table4:</b>
@@ -171,15 +171,15 @@ EOT
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
   <tr>
     <td><pre>group1</pre></td>
-    <td><pre>'val1'</pre></td>
+    <td><pre>&#039;val1&#039;</pre></td>
-    <td><pre>'val2'</pre></td>
+    <td><pre>&#039;val2&#039;</pre></td>
-    <td><pre>'val3'</pre></td>
+    <td><pre>&#039;val3&#039;</pre></td>
   </tr>
 </table>
 

test/python/cli/test_cmd_import.py

@@ -40,7 +40,6 @@ class TestCliImportWithDb:
             mock_func_factory(nominatim.data.country_info, 'setup_country_tables'),
             mock_func_factory(nominatim.tools.database_import, 'import_osm_data'),
             mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
-            mock_func_factory(nominatim.tools.refresh, 'import_secondary_importance'),
             mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
             mock_func_factory(nominatim.tools.database_import, 'load_data'),
             mock_func_factory(nominatim.tools.database_import, 'create_tables'),
@@ -106,8 +105,11 @@ class TestCliImportWithDb:
         for mock in mocks:
             assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
 
+        assert temp_db_conn.index_exists('idx_placex_pendingsector')
+
         # Calling it again still works for the index
         assert self.call_nominatim('import', '--continue', 'indexing') == 0
+        assert temp_db_conn.index_exists('idx_placex_pendingsector')
 
 
     def test_import_continue_postprocess(self, mock_func_factory):