Merge pull request #3747 from anqixxx/fix-special-phrases-filtering

Special Phrases Filtering: Add Command Line Functionality

.flake8

@@ -6,3 +6,6 @@ extend-ignore =
     E711
 per-file-ignores =
     __init__.py: F401
+    test/python/utils/test_json_writer.py: E131
+    **/conftest.py: E402
+    test/bdd/*: F821

.github/actions/setup-postgresql/action.yml

@@ -11,10 +11,8 @@ runs:
     steps:
         - name: Remove existing PostgreSQL
           run: |
+              sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
               sudo apt-get purge -yq postgresql*
-              sudo apt install curl ca-certificates gnupg
-              curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
-              sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
               sudo apt-get update -qq
 
           shell: bash

.github/workflows/ci-tests.yml

@@ -37,10 +37,10 @@ jobs:
         needs: create-archive
         strategy:
             matrix:
-                flavour: ["ubuntu-20", "ubuntu-24"]
+                flavour: ["ubuntu-22", "ubuntu-24"]
                 include:
-                    - flavour: ubuntu-20
-                      ubuntu: 20
+                    - flavour: ubuntu-22
+                      ubuntu: 22
                       postgresql: 12
                       lua: '5.1'
                       dependencies: pip
@@ -68,26 +68,34 @@ jobs:
               with:
                   dependencies: ${{ matrix.dependencies }}
 
+            - uses: actions/cache@v4
+              with:
+                  path: |
+                     /usr/local/bin/osm2pgsql
+                  key: osm2pgsql-bin-22-1
+              if: matrix.ubuntu == '22'
+
             - name: Compile osm2pgsql
               run: |
-                  sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua-dkjson nlohmann-json3-dev 
-                  mkdir osm2pgsql-build
-                  cd osm2pgsql-build
-                  git clone https://github.com/osm2pgsql-dev/osm2pgsql
-                  mkdir build
-                  cd build
-                  cmake ../osm2pgsql
-                  make
-                  sudo make install
-                  cd ../..
-                  rm -rf osm2pgsql-build
-              if: matrix.ubuntu == '20'
+                  if [ ! -f /usr/local/bin/osm2pgsql ]; then
+                      sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua-dkjson nlohmann-json3-dev 
+                      mkdir osm2pgsql-build
+                      cd osm2pgsql-build
+                      git clone https://github.com/osm2pgsql-dev/osm2pgsql
+                      mkdir build
+                      cd build
+                      cmake ../osm2pgsql
+                      make
+                      sudo make install
+                      cd ../..
+                      rm -rf osm2pgsql-build
+                  else
+                      sudo apt-get install -y -qq libexpat1 liblua${LUA_VERSION}
+                  fi
+              if: matrix.ubuntu == '22'
               env:
                   LUA_VERSION: ${{ matrix.lua }}
 
-            - name: Install test prerequisites
-              run: ./venv/bin/pip install behave==1.2.6
-
             - name: Install test prerequisites (apt)
               run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn python3-falcon python3-aiosqlite python3-pyosmium
               if: matrix.dependencies == 'apt'
@@ -96,11 +104,14 @@ jobs:
               run: ./venv/bin/pip install pytest-asyncio falcon starlette asgi_lifespan aiosqlite osmium uvicorn
               if: matrix.dependencies == 'pip'
 
+            - name: Install test prerequisites
+              run: ./venv/bin/pip install pytest-bdd
+
             - name: Install latest flake8
               run: ./venv/bin/pip install -U flake8
 
             - name: Python linting
-              run: ../venv/bin/python -m flake8 src
+              run: ../venv/bin/python -m flake8 src test/python test/bdd
               working-directory: Nominatim
 
             - name: Install mypy and typechecking info
@@ -118,8 +129,8 @@ jobs:
 
             - name: BDD tests
               run: |
-                  ../../../venv/bin/python -m behave -DREMOVE_TEMPLATE=1 --format=progress3
-              working-directory: Nominatim/test/bdd
+                  ../venv/bin/python -m pytest test/bdd --nominatim-purge
+              working-directory: Nominatim
 
     install:
         runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -113,3 +113,5 @@ Checklist for releases:
   * run `nominatim --version` to confirm correct version
 * [ ] tag new release and add a release on github.com
 * [ ] build pip packages and upload to pypi
+  * `make build`
+  * `twine upload dist/*`

ChangeLog

@@ -1,3 +1,22 @@
+5.1.0
+ * replace datrie with simple internal trie implementation
+ * add pattern-based postcode parser for queries,
+   postcodes no longer need to be present in OSM to be found
+ * take variants into account when computing token similarity
+ * add extratags output to geocodejson format
+ * fix default layer setting used for structured queries
+ * update abbreviation lists for Russian and English
+   (thanks @shoorick, @IvanShift, @mhsrn21)
+ * fix variant generation for Norwegian
+ * fix normalization around space-like characters
+ * improve postcode search and handling of postcodes in queries
+ * reorganise internal query structure and get rid of slow enums
+ * enable code linting for tests
+ * various code moderinsations in test code (thanks @eumiro)
+ * remove setting osm2pgsql location via config.lib_dir
+ * make SQL functions parallel save as far as possible (thanks @otbutz)
+ * various fixes and improvements to documentation (thanks @TuringVerified)
+
 5.0.0
  * increase required versions for PostgreSQL (12+), PostGIS (3.0+)
  * remove installation via cmake and debundle osm2pgsql

Makefile

@@ -24,10 +24,10 @@ pytest:
 	pytest test/python
 
 lint:
-	flake8 src
+	flake8 src test/python test/bdd
 
 bdd:
-	cd test/bdd; behave -DREMOVE_TEMPLATE=1
+	pytest test/bdd --nominatim-purge
 
 # Documentation
 

README.md

@@ -27,18 +27,25 @@ can be found at nominatim.org as well.
 
 A quick summary of the necessary steps:
 
-1. Create a Python virtualenv and install the packages:
+
+1. Clone this git repository and download the country grid
+
+        git clone https://github.com/osm-search/Nominatim.git
+        wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
+
+2. Create a Python virtualenv and install the packages:
 
         python3 -m venv nominatim-venv
         ./nominatim-venv/bin/pip install packaging/nominatim-{api,db}
 
-2. Create a project directory, get OSM data and import:
+3. Create a project directory, get OSM data and import:
 
         mkdir nominatim-project
         cd nominatim-project
-        ../nominatim-venv/bin/nominatim import --osm-file <your planet file>
+        ../nominatim-venv/bin/nominatim import --osm-file <your planet file> 2>&1 | tee setup.log
 
-3. Start the webserver:
+
+4. Start the webserver:
 
         ./nominatim-venv/bin/pip install uvicorn falcon
         ../nominatim-venv/bin/nominatim serve

SECURITY.md

@@ -9,7 +9,8 @@ versions.
 
 | Version | End of support for security updates |
 | ------- | ----------------------------------- |
-| 5.0.x   | 2027-02-06
+| 5.1.x   | 2027-04-01                          |
+| 5.0.x   | 2027-02-06                          |
 | 4.5.x   | 2026-09-12                          |
 | 4.4.x   | 2026-03-07                          |
 | 4.3.x   | 2025-09-07                          |

docs/admin/Installation.md

@@ -37,7 +37,6 @@ Furthermore the following Python libraries are required:
   * [Jinja2](https://palletsprojects.com/p/jinja/)
   * [PyICU](https://pypi.org/project/PyICU/)
   * [PyYaml](https://pyyaml.org/) (5.1+)
-  * [datrie](https://github.com/pytries/datrie)
 
 These will be installed automatically when using pip installation.
 

docs/api/Output.md

@@ -106,8 +106,11 @@ The following feature attributes are implemented:
  * `name` - localised name of the place
  * `housenumber`, `street`, `locality`, `district`, `postcode`, `city`,
    `county`, `state`, `country` -
-   provided when it can be determined from the address
+   provided when it can be determined from the address (only with `addressdetails=1`)
  * `admin` - list of localised names of administrative boundaries (only with `addressdetails=1`)
+ * `extra` - dictionary with additional useful tags like `website` or `maxspeed`
+   (only with `extratags=1`)
+
 
 Use `polygon_geojson` to output the full geometry of the object instead
 of the centroid.

docs/api/Search.md

@@ -212,7 +212,7 @@ other layers.
 The featureType allows to have a more fine-grained selection for places
 from the address layer. Results can be restricted to places that make up
 the 'state', 'country' or 'city' part of an address. A featureType of
-settlement selects any human inhabited feature from 'state' down to
+`settlement` selects any human inhabited feature from 'state' down to
 'neighbourhood'.
 
 When featureType is set, then results are automatically restricted

docs/customize/Settings.md

@@ -602,6 +602,43 @@ results gathered so far.
 Note that under high load you may observe that users receive different results
 than usual without seeing an error. This may cause some confusion.
 
+#### NOMINATIM_OUTPUT_NAMES
+
+| Summary            |                                                     |
+| --------------     | --------------------------------------------------- |
+| **Description:**   | Specifies order of name tags |
+| **Format:**        | string: comma-separated list of tag names |
+| **Default:**       | name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref |
+
+Specifies the order in which different name tags are used.
+The values in this list determine the preferred order of name variants,
+including language-specific names (in OSM: the name tag with and without any language suffix).
+
+Comma-separated list, where :XX stands for language suffix
+(e.g. name:en) and no :XX stands for general tags (e.g. name).
+
+See also [NOMINATIM_DEFAULT_LANGUAGE](#nominatim_default_language).
+
+!!! note
+    If NOMINATIM_OUTPUT_NAMES = `name:XX,name,short_name:XX,short_name` the search follows
+
+        ```
+        'name', 'short_name'
+        ```
+
+    if we have no preferred language order for showing search results.
+
+    For languages ['en', 'es'] the search follows
+
+        ```
+        'name:en', 'name:es',
+        'name',
+        'short_name:en', 'short_name:es',
+        'short_name'
+        ```
+
+    For those familiar with the internal implementation, the `_place_*` expansion is added, but to simplify, it is not included in this example.
+
 ### Logging Settings
 
 #### NOMINATIM_LOG_DB

docs/customize/Tokenizers.md

@@ -67,7 +67,13 @@ Here is an example configuration file:
 
 ``` yaml
 query-preprocessing:
-    - normalize
+    - step: split_japanese_phrases
+    - step: regex_replace
+      replacements:
+        - pattern: https?://[^\s]* # Filter URLs starting with http or https
+          replace: ''
+    - step: normalize
+
 normalization:
     - ":: lower ()"
     - "ß > 'ss'" # German szet is unambiguously equal to double ss
@@ -88,8 +94,8 @@ token-analysis:
             replacements: ['ä', 'ae']
 ```
 
-The configuration file contains four sections:
-`normalization`, `transliteration`, `sanitizers` and `token-analysis`.
+The configuration file contains five sections:
+`query-preprocessing`, `normalization`, `transliteration`, `sanitizers` and `token-analysis`.
 
 #### Query preprocessing
 
@@ -106,6 +112,19 @@ The following is a list of preprocessors that are shipped with Nominatim.
         heading_level: 6
         docstring_section_style: spacy
 
+##### regex-replace
+
+::: nominatim_api.query_preprocessing.regex_replace
+    options:
+        members: False
+        heading_level: 6
+        docstring_section_style: spacy
+    description: 
+        This option runs any given regex pattern on the input and replaces values accordingly
+    replacements:
+        - pattern: regex pattern
+          replace: string to replace with
+
 
 #### Normalization and Transliteration
 

docs/develop/Development-Environment.md

@@ -25,15 +25,15 @@ following packages should get you started:
 
 ## Prerequisites for testing and documentation
 
-The Nominatim test suite consists of behavioural tests (using behave) and
+The Nominatim test suite consists of behavioural tests (using pytest-bdd) and
 unit tests (using pytest). It has the following additional requirements:
 
-* [behave test framework](https://behave.readthedocs.io) >= 1.2.6
 * [flake8](https://flake8.pycqa.org/en/stable/) (CI always runs the latest version from pip)
 * [mypy](http://mypy-lang.org/) (plus typing information for external libs)
 * [Python Typing Extensions](https://github.com/python/typing_extensions) (for Python < 3.9)
 * [pytest](https://pytest.org)
 * [pytest-asyncio](https://pytest-asyncio.readthedocs.io)
+* [pytest-bdd](https://pytest-bdd.readthedocs.io)
 
 For testing the Python search frontend, you need to install extra dependencies
 depending on your choice of webserver framework:
@@ -48,9 +48,6 @@ The documentation is built with mkdocs:
 * [mkdocs-material](https://squidfunk.github.io/mkdocs-material/)
 * [mkdocs-gen-files](https://oprypin.github.io/mkdocs-gen-files/)
 
-Please be aware that tests always run against the globally installed
-osm2pgsql, so you need to have this set up. If you want to test against
-the vendored version of osm2pgsql, you need to set the PATH accordingly.
 
 ### Installing prerequisites on Ubuntu/Debian
 
@@ -69,9 +66,10 @@ To set up the virtual environment with all necessary packages run:
 ```sh
 virtualenv ~/nominatim-dev-venv
 ~/nominatim-dev-venv/bin/pip install\
-    psutil psycopg[binary] PyICU SQLAlchemy \
-    python-dotenv jinja2 pyYAML datrie behave \
-    mkdocs mkdocstrings mkdocs-gen-files pytest pytest-asyncio flake8 \
+    psutil 'psycopg[binary]' PyICU SQLAlchemy \
+    python-dotenv jinja2 pyYAML \
+    mkdocs 'mkdocstrings[python]' mkdocs-gen-files \
+    pytest pytest-asyncio pytest-bdd flake8 \
     types-jinja2 types-markupsafe types-psutil types-psycopg2 \
     types-pygments types-pyyaml types-requests types-ujson \
     types-urllib3 typing-extensions unicorn falcon starlette \

docs/develop/ICU-Tokenizer-Modules.md

@@ -60,13 +60,19 @@ The order of phrases matters to Nominatim when doing further processing.
 Thus, while you may split or join phrases, you should not reorder them
 unless you really know what you are doing.
 
-Phrase types (`nominatim_api.search.PhraseType`) can further help narrowing
-down how the tokens in the phrase are interpreted. The following phrase types
-are known:
+Phrase types can further help narrowing down how the tokens in the phrase
+are interpreted. The following phrase types are known:
 
-::: nominatim_api.search.PhraseType
-    options:
-        heading_level: 6
+| Name           | Description |
+|----------------|-------------|
+| PHRASE_ANY     | No specific designation (i.e. source is free-form query) |
+| PHRASE_AMENITY | Contains name or type of a POI |
+| PHRASE_STREET  | Contains a street name optionally with a housenumber |
+| PHRASE_CITY    | Contains the postal city |
+| PHRASE_COUNTY  | Contains the equivalent of a county |
+| PHRASE_STATE   | Contains a state or province |
+| PHRASE_POSTCODE| Contains a postal code |
+| PHRASE_COUNTRY | Contains the country name or code |
 
 
 ## Custom sanitizer modules

docs/develop/Testing.md

@@ -43,53 +43,53 @@ The name of the pytest binary depends on your installation.
 ## BDD Functional Tests (`test/bdd`)
 
 Functional tests are written as BDD instructions. For more information on
-the philosophy of BDD testing, see the
-[Behave manual](http://pythonhosted.org/behave/philosophy.html).
-
-The following explanation assume that the reader is familiar with the BDD
-notations of features, scenarios and steps.
-
-All possible steps can be found in the `steps` directory and should ideally
-be documented.
+the philosophy of BDD testing, read the Wikipedia article on
+[Behaviour-driven development](https://en.wikipedia.org/wiki/Behavior-driven_development).
 
 ### General Usage
 
 To run the functional tests, do
 
-    cd test/bdd
-    behave
+    pytest test/bdd
 
-The tests can be configured with a set of environment variables (`behave -D key=val`):
+The BDD tests create databases for the tests. You can set name of the databases
+through configuration variables in your `pytest.ini`:
 
- * `TEMPLATE_DB` - name of template database used as a skeleton for
-                   the test databases (db tests)
- * `TEST_DB` - name of test database (db tests)
- * `API_TEST_DB` - name of the database containing the API test data (api tests)
- * `API_TEST_FILE` - OSM file to be imported into the API test database (api tests)
- * `API_ENGINE` - webframe to use for running search queries, same values as
-                  `nominatim serve --engine` parameter
- * `DB_HOST` - (optional) hostname of database host
- * `DB_PORT` - (optional) port of database on host
- * `DB_USER` - (optional) username of database login
- * `DB_PASS` - (optional) password for database login
- * `REMOVE_TEMPLATE` - if true, the template and API database will not be reused
-                       during the next run. Reusing the base templates speeds
-                       up tests considerably but might lead to outdated errors
-                       for some changes in the database layout.
- * `KEEP_TEST_DB` - if true, the test database will not be dropped after a test
-                    is finished. Should only be used if one single scenario is
-                    run, otherwise the result is undefined.
+ * `nominatim_test_db` defines the name of the temporary database created for
+    a single test (default: `test_nominatim`)
+ * `nominatim_api_test_db` defines the name of the database containing
+    the API test data, see also below (default: `test_api_nominatim`)
+ * `nominatim_template_db` defines the name of the template database used
+    for creating the temporary test databases. It contains some static setup
+    which usually doesn't change between imports of OSM data
+    (default: `test_template_nominatim`)
+
+To change other connection parameters for the PostgreSQL database, use
+the [libpq enivronment variables](https://www.postgresql.org/docs/current/libpq-envars.html).
+Never set a password through these variables. Use a
+[password file](https://www.postgresql.org/docs/current/libpq-pgpass.html) instead.
+
+The API test database and the template database are only created once and then
+left untouched. This is usually what you want because it speeds up subsequent
+runs of BDD tests. If you do change code that has an influence on the content
+of these databases, you can run pytest with the `--nominatim-purge` parameter
+and the databases will be dropped and recreated from scratch.
+
+When running the BDD tests with make (using `make tests` or `make bdd`), then
+the databases will always be purged.
+
+The temporary test database is usually dropped directly after the test, so
+it does not take up unnecessary space. If you want to keep the database around,
+for example while debugging a specific BDD test, use the parameter
+`--nominatim-keep-db`.
 
-Logging can be defined through command line parameters of behave itself. Check
-out `behave --help` for details. Also have a look at the 'work-in-progress'
-feature of behave which comes in handy when writing new tests.
 
 ### API Tests (`test/bdd/api`)
 
 These tests are meant to test the different API endpoints and their parameters.
 They require to import several datasets into a test database. This is normally
 done automatically during setup of the test. The API test database is then
-kept around and reused in subsequent runs of behave. Use `behave -DREMOVE_TEMPLATE`
+kept around and reused in subsequent runs of behave. Use `--nominatim-purge`
 to force a reimport of the database.
 
 The official test dataset is saved in the file `test/testdb/apidb-test-data.pbf`
@@ -109,12 +109,12 @@ test the correctness of osm2pgsql. Each test will write some data into the `plac
 table (and optionally the `planet_osm_*` tables if required) and then run
 Nominatim's processing functions on that.
 
-These tests need to create their own test databases. By default they will be
-called `test_template_nominatim` and `test_nominatim`. Names can be changed with
-the environment variables `TEMPLATE_DB` and `TEST_DB`. The user running the tests
-needs superuser rights for postgres.
+These tests use the template database and create temporary test databases for
+each test.
 
 ### Import Tests (`test/bdd/osm2pgsql`)
 
-These tests check that data is imported correctly into the place table. They
-use the same template database as the DB Creation tests, so the same remarks apply.
+These tests check that data is imported correctly into the place table.
+
+These tests also use the template database and create temporary test databases
+for each test.

lib-lua/themes/nominatim/presets.lua

@@ -187,7 +187,7 @@ module.MAIN_TAGS_POIS = function (group)
                passing_place = group,
                street_lamp = 'named',
                traffic_signals = 'named'},
-    historic = {'always',
+    historic = {'fallback',
                 yes = group,
                 no = group},
     information = {include_when_tag_present('tourism', 'information'),
@@ -196,6 +196,7 @@ module.MAIN_TAGS_POIS = function (group)
                    trail_blaze = 'never'},
     junction = {'fallback',
                 no = group},
+    landuse = {cemetery = 'always'},
     leisure = {'always',
                nature_reserve = 'fallback',
                swimming_pool = 'named',
@@ -229,6 +230,7 @@ module.MAIN_TAGS_POIS = function (group)
     shop = {'always',
             no = group},
     tourism = {'always',
+               attraction = 'fallback',
                no = group,
                yes = group,
                information = exclude_when_key_present('information')},
@@ -330,7 +332,7 @@ module.NAME_TAGS.core = {main = {'name', 'name:*',
                         }
 module.NAME_TAGS.address = {house = {'addr:housename'}}
 module.NAME_TAGS.poi = group_merge({main = {'brand'},
-                                    extra = {'iata', 'icao'}},
+                                    extra = {'iata', 'icao', 'faa'}},
                                    module.NAME_TAGS.core)
 
 -- Address tagging

lib-sql/functions.sql

@@ -8,7 +8,6 @@
 {% include('functions/utils.sql') %}
 {% include('functions/ranking.sql') %}
 {% include('functions/importance.sql') %}
-{% include('functions/address_lookup.sql') %}
 {% include('functions/interpolation.sql') %}
 
 {% if 'place' in db.tables %}

lib-sql/functions/address_lookup.sql

@@ -1,334 +0,0 @@
--- SPDX-License-Identifier: GPL-2.0-only
---
--- This file is part of Nominatim. (https://nominatim.org)
---
--- Copyright (C) 2022 by the Nominatim developer community.
--- For a full list of authors see the git log.
-
--- Functions for returning address information for a place.
-
-DROP TYPE IF EXISTS addressline CASCADE;
-CREATE TYPE addressline as (
-  place_id BIGINT,
-  osm_type CHAR(1),
-  osm_id BIGINT,
-  name HSTORE,
-  class TEXT,
-  type TEXT,
-  place_type TEXT,
-  admin_level INTEGER,
-  fromarea BOOLEAN,
-  isaddress BOOLEAN,
-  rank_address INTEGER,
-  distance FLOAT
-);
-
-
-CREATE OR REPLACE FUNCTION get_name_by_language(name hstore, languagepref TEXT[])
-  RETURNS TEXT
-  AS $$
-DECLARE
-  result TEXT;
-BEGIN
-  IF name is null THEN
-    RETURN null;
-  END IF;
-
-  FOR j IN 1..array_upper(languagepref,1) LOOP
-    IF name ? languagepref[j] THEN
-      result := trim(name->languagepref[j]);
-      IF result != '' THEN
-        return result;
-      END IF;
-    END IF;
-  END LOOP;
-
-  -- as a fallback - take the last element since it is the default name
-  RETURN trim((avals(name))[array_length(avals(name), 1)]);
-END;
-$$
-LANGUAGE plpgsql IMMUTABLE;
-
-
---housenumber only needed for tiger data
-CREATE OR REPLACE FUNCTION get_address_by_language(for_place_id BIGINT,
-                                                   housenumber INTEGER,
-                                                   languagepref TEXT[])
-  RETURNS TEXT
-  AS $$
-DECLARE
-  result TEXT[];
-  currresult TEXT;
-  prevresult TEXT;
-  location RECORD;
-BEGIN
-
-  result := '{}';
-  prevresult := '';
-
-  FOR location IN
-    SELECT name,
-       CASE WHEN place_id = for_place_id THEN 99 ELSE rank_address END as rank_address
-    FROM get_addressdata(for_place_id, housenumber)
-    WHERE isaddress order by rank_address desc
-  LOOP
-    currresult := trim(get_name_by_language(location.name, languagepref));
-    IF currresult != prevresult AND currresult IS NOT NULL
-       AND result[(100 - location.rank_address)] IS NULL
-    THEN
-      result[(100 - location.rank_address)] := currresult;
-      prevresult := currresult;
-    END IF;
-  END LOOP;
-
-  RETURN array_to_string(result,', ');
-END;
-$$
-LANGUAGE plpgsql STABLE;
-
-DROP TYPE IF EXISTS addressdata_place;
-CREATE TYPE addressdata_place AS (
-  place_id BIGINT,
-  country_code VARCHAR(2),
-  housenumber TEXT,
-  postcode TEXT,
-  class TEXT,
-  type TEXT,
-  name HSTORE,
-  address HSTORE,
-  centroid GEOMETRY
-);
-
--- Compute the list of address parts for the given place.
---
--- If in_housenumber is greator or equal 0, look for an interpolation.
-CREATE OR REPLACE FUNCTION get_addressdata(in_place_id BIGINT, in_housenumber INTEGER)
-  RETURNS setof addressline
-  AS $$
-DECLARE
-  place addressdata_place;
-  location RECORD;
-  country RECORD;
-  current_rank_address INTEGER;
-  location_isaddress BOOLEAN;
-BEGIN
-  -- The place in question might not have a direct entry in place_addressline.
-  -- Look for the parent of such places then and save it in place.
-
-  -- first query osmline (interpolation lines)
-  IF in_housenumber >= 0 THEN
-    SELECT parent_place_id as place_id, country_code,
-           in_housenumber as housenumber, postcode,
-           'place' as class, 'house' as type,
-           null as name, null as address,
-           ST_Centroid(linegeo) as centroid
-      INTO place
-      FROM location_property_osmline
-      WHERE place_id = in_place_id
-            AND in_housenumber between startnumber and endnumber;
-  END IF;
-
-  --then query tiger data
-  {% if config.get_bool('USE_US_TIGER_DATA') %}
-  IF place IS NULL AND in_housenumber >= 0 THEN
-    SELECT parent_place_id as place_id, 'us' as country_code,
-           in_housenumber as housenumber, postcode,
-           'place' as class, 'house' as type,
-           null as name, null as address,
-           ST_Centroid(linegeo) as centroid
-      INTO place
-      FROM location_property_tiger
-      WHERE place_id = in_place_id
-            AND in_housenumber between startnumber and endnumber;
-  END IF;
-  {% endif %}
-
-  -- postcode table
-  IF place IS NULL THEN
-    SELECT parent_place_id as place_id, country_code,
-           null::text as housenumber, postcode,
-           'place' as class, 'postcode' as type,
-           null as name, null as address,
-           null as centroid
-      INTO place
-      FROM location_postcode
-      WHERE place_id = in_place_id;
-  END IF;
-
-  -- POI objects in the placex table
-  IF place IS NULL THEN
-    SELECT parent_place_id as place_id, country_code,
-           coalesce(address->'housenumber',
-                    address->'streetnumber',
-                    address->'conscriptionnumber')::text as housenumber,
-           postcode,
-           class, type,
-           name, address,
-           centroid
-      INTO place
-      FROM placex
-      WHERE place_id = in_place_id and rank_search > 27;
-  END IF;
-
-  -- If place is still NULL at this point then the object has its own
-  -- entry in place_address line. However, still check if there is not linked
-  -- place we should be using instead.
-  IF place IS NULL THEN
-    select coalesce(linked_place_id, place_id) as place_id,  country_code,
-           null::text as housenumber, postcode,
-           class, type,
-           null as name, address,
-           null as centroid
-      INTO place
-      FROM placex where place_id = in_place_id;
-  END IF;
-
---RAISE WARNING '% % % %',searchcountrycode, searchhousenumber, searchpostcode;
-
-  -- --- Return the record for the base entry.
-
-  current_rank_address := 1000;
-  FOR location IN
-    SELECT placex.place_id, osm_type, osm_id, name,
-           coalesce(extratags->'linked_place', extratags->'place') as place_type,
-           class, type, admin_level,
-           CASE WHEN rank_address = 0 THEN 100
-                WHEN rank_address = 11 THEN 5
-                ELSE rank_address END as rank_address,
-           country_code
-      FROM placex
-      WHERE place_id = place.place_id
-  LOOP
---RAISE WARNING '%',location;
-    -- mix in default names for countries
-    IF location.rank_address = 4 and place.country_code is not NULL THEN
-      FOR country IN
-        SELECT coalesce(name, ''::hstore) as name FROM country_name
-          WHERE country_code = place.country_code LIMIT 1
-      LOOP
-        place.name := country.name || place.name;
-      END LOOP;
-    END IF;
-
-    IF location.rank_address < 4 THEN
-      -- no country locations for ranks higher than country
-      place.country_code := NULL::varchar(2);
-    ELSEIF place.country_code IS NULL AND location.country_code IS NOT NULL THEN
-      place.country_code := location.country_code;
-    END IF;
-
-    RETURN NEXT ROW(location.place_id, location.osm_type, location.osm_id,
-                    location.name, location.class, location.type,
-                    location.place_type,
-                    location.admin_level, true,
-                    location.type not in ('postcode', 'postal_code'),
-                    location.rank_address, 0)::addressline;
-
-    current_rank_address := location.rank_address;
-  END LOOP;
-
-  -- --- Return records for address parts.
-
-  FOR location IN
-    SELECT placex.place_id, osm_type, osm_id, name, class, type,
-           coalesce(extratags->'linked_place', extratags->'place') as place_type,
-           admin_level, fromarea, isaddress,
-           CASE WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address,
-           distance, country_code, postcode
-      FROM place_addressline join placex on (address_place_id = placex.place_id)
-      WHERE place_addressline.place_id IN (place.place_id, in_place_id)
-            AND linked_place_id is null
-            AND (placex.country_code IS NULL OR place.country_code IS NULL
-                 OR placex.country_code = place.country_code)
-      ORDER BY rank_address desc,
-               (place_addressline.place_id = in_place_id) desc,
-               (CASE WHEN coalesce((avals(name) && avals(place.address)), False) THEN 2
-                     WHEN isaddress THEN 0
-                     WHEN fromarea
-                          and place.centroid is not null
-                          and ST_Contains(geometry, place.centroid) THEN 1
-                     ELSE -1 END) desc,
-               fromarea desc, distance asc, rank_search desc
-  LOOP
-    -- RAISE WARNING '%',location;
-    location_isaddress := location.rank_address != current_rank_address;
-
-    IF place.country_code IS NULL AND location.country_code IS NOT NULL THEN
-      place.country_code := location.country_code;
-    END IF;
-    IF location.type in ('postcode', 'postal_code')
-       AND place.postcode is not null
-    THEN
-      -- If the place had a postcode assigned, take this one only
-      -- into consideration when it is an area and the place does not have
-      -- a postcode itself.
-      IF location.fromarea AND location_isaddress
-         AND (place.address is null or not place.address ? 'postcode')
-      THEN
-        place.postcode := null; -- remove the less exact postcode
-      ELSE
-        location_isaddress := false;
-      END IF;
-    END IF;
-    RETURN NEXT ROW(location.place_id, location.osm_type, location.osm_id,
-                    location.name, location.class, location.type,
-                    location.place_type,
-                    location.admin_level, location.fromarea,
-                    location_isaddress,
-                    location.rank_address,
-                    location.distance)::addressline;
-
-    current_rank_address := location.rank_address;
-  END LOOP;
-
-  -- If no country was included yet, add the name information from country_name.
-  IF current_rank_address > 4 THEN
-    FOR location IN
-      SELECT name || coalesce(derived_name, ''::hstore) as name FROM country_name
-        WHERE country_code = place.country_code LIMIT 1
-    LOOP
---RAISE WARNING '% % %',current_rank_address,searchcountrycode,countryname;
-      RETURN NEXT ROW(null, null, null, location.name, 'place', 'country', NULL,
-                      null, true, true, 4, 0)::addressline;
-    END LOOP;
-  END IF;
-
-  -- Finally add some artificial rows.
-  IF place.country_code IS NOT NULL THEN
-    location := ROW(null, null, null, hstore('ref', place.country_code),
-                    'place', 'country_code', null, null, true, false, 4, 0)::addressline;
-    RETURN NEXT location;
-  END IF;
-
-  IF place.name IS NOT NULL THEN
-    location := ROW(in_place_id, null, null, place.name, place.class,
-                    place.type, null, null, true, true, 29, 0)::addressline;
-    RETURN NEXT location;
-  END IF;
-
-  IF place.housenumber IS NOT NULL THEN
-    location := ROW(null, null, null, hstore('ref', place.housenumber),
-                    'place', 'house_number', null, null, true, true, 28, 0)::addressline;
-    RETURN NEXT location;
-  END IF;
-
-  IF place.address is not null and place.address ? '_unlisted_place' THEN
-    RETURN NEXT ROW(null, null, null, hstore('name', place.address->'_unlisted_place'),
-                    'place', 'locality', null, null, true, true, 25, 0)::addressline;
-  END IF;
-
-  IF place.postcode is not null THEN
-    location := ROW(null, null, null, hstore('ref', place.postcode), 'place',
-                    'postcode', null, null, false, true, 5, 0)::addressline;
-    RETURN NEXT location;
-  ELSEIF place.address is not null and place.address ? 'postcode'
-         and not place.address->'postcode' SIMILAR TO '%(,|;)%' THEN
-    location := ROW(null, null, null, hstore('ref', place.address->'postcode'), 'place',
-                    'postcode', null, null, false, true, 5, 0)::addressline;
-    RETURN NEXT location;
-  END IF;
-
-  RETURN;
-END;
-$$
-LANGUAGE plpgsql STABLE;

lib-sql/functions/importance.sql

@@ -65,7 +65,7 @@ BEGIN
   RETURN NULL;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 {% else %}
 
@@ -78,7 +78,7 @@ SELECT convert_from(CAST(E'\\x' || array_to_string(ARRAY(
     FROM regexp_matches($1, '%[0-9a-f][0-9a-f]|.', 'gi') AS r(m)
 ), '') AS bytea), 'UTF8');
 $$ 
-LANGUAGE SQL IMMUTABLE STRICT;
+LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION catch_decode_url_part(p varchar)
@@ -91,7 +91,7 @@ EXCEPTION
   WHEN others THEN return null;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE STRICT;
+LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION get_wikipedia_match(extratags HSTORE, country_code varchar(2))
@@ -139,7 +139,7 @@ BEGIN
   RETURN NULL;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 {% endif %}
 
@@ -203,5 +203,5 @@ BEGIN
   RETURN result;
 END;
 $$
-LANGUAGE plpgsql;
+LANGUAGE plpgsql PARALLEL SAFE;
 

lib-sql/functions/interpolation.sql

@@ -34,7 +34,7 @@ BEGIN
   RETURN in_address;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 
@@ -70,7 +70,7 @@ BEGIN
   RETURN parent_place_id;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION reinsert_interpolation(way_id BIGINT, addr HSTORE,

lib-sql/functions/partition-functions.sql

@@ -58,7 +58,7 @@ BEGIN
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION get_address_place(in_partition SMALLINT, feature GEOMETRY,
@@ -87,7 +87,7 @@ BEGIN
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 create or replace function deleteLocationArea(in_partition INTEGER, in_place_id BIGINT, in_rank_search INTEGER) RETURNS BOOLEAN AS $$
@@ -172,7 +172,7 @@ BEGIN
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 CREATE OR REPLACE FUNCTION getNearestNamedPlacePlaceId(in_partition INTEGER,
                                                        point GEOMETRY,
@@ -202,7 +202,7 @@ BEGIN
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 create or replace function insertSearchName(
   in_partition INTEGER, in_place_id BIGINT, in_name_vector INTEGER[],
@@ -310,7 +310,7 @@ BEGIN
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 CREATE OR REPLACE FUNCTION getNearestParallelRoadFeature(in_partition INTEGER,
                                                          line GEOMETRY)
@@ -354,4 +354,4 @@ BEGIN
   RAISE EXCEPTION 'Unknown partition %', in_partition;
 END
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;

lib-sql/functions/placex_triggers.sql

@@ -109,7 +109,7 @@ BEGIN
   RETURN result;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION find_associated_street(poi_osm_type CHAR(1),
@@ -200,7 +200,7 @@ BEGIN
   RETURN result;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 -- Find the parent road of a POI.
@@ -286,7 +286,7 @@ BEGIN
   RETURN parent_place_id;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 -- Try to find a linked place for the given object.
 CREATE OR REPLACE FUNCTION find_linked_place(bnd placex)
@@ -404,7 +404,7 @@ BEGIN
   RETURN NULL;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION create_poi_search_terms(obj_place_id BIGINT,

lib-sql/functions/ranking.sql

@@ -29,7 +29,7 @@ BEGIN
   RETURN 0.02;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 -- Return an approximate update radius according to the search rank.
@@ -60,7 +60,7 @@ BEGIN
   RETURN 0;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 -- Compute a base address rank from the extent of the given geometry.
 --
@@ -107,7 +107,7 @@ BEGIN
    RETURN 23;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 -- Guess a ranking for postcodes from country and postcode format.
@@ -167,7 +167,7 @@ BEGIN
 
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 -- Get standard search and address rank for an object.
@@ -236,7 +236,7 @@ BEGIN
   END IF;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 CREATE OR REPLACE FUNCTION get_addr_tag_rank(key TEXT, country TEXT,
                                              OUT from_rank SMALLINT,
@@ -283,7 +283,7 @@ BEGIN
   END LOOP;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION weigh_search(search_vector INT[],
@@ -304,4 +304,4 @@ BEGIN
   RETURN def_weight;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;

lib-sql/functions/utils.sql

@@ -24,7 +24,7 @@ BEGIN
   RETURN ST_PointOnSurface(place);
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION geometry_sector(partition INTEGER, place GEOMETRY)
@@ -34,7 +34,7 @@ BEGIN
   RETURN (partition*1000000) + (500-ST_X(place)::INTEGER)*1000 + (500-ST_Y(place)::INTEGER);
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 
@@ -60,7 +60,7 @@ BEGIN
   RETURN r;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 -- Return the node members with a given label from a relation member list
 -- as a set.
@@ -88,7 +88,7 @@ BEGIN
   RETURN;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION get_rel_node_members(members JSONB, memberLabels TEXT[])
@@ -107,7 +107,7 @@ BEGIN
   RETURN;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 -- Copy 'name' to or from the default language.
@@ -136,7 +136,7 @@ BEGIN
   END IF;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 -- Find the nearest artificial postcode for the given geometry.
@@ -172,7 +172,7 @@ BEGIN
     RETURN outcode;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION get_country_code(place geometry)
@@ -233,7 +233,7 @@ BEGIN
   RETURN NULL;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION get_country_language_code(search_country_code VARCHAR(2))
@@ -251,7 +251,7 @@ BEGIN
   RETURN NULL;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION get_partition(in_country_code VARCHAR(10))
@@ -268,7 +268,7 @@ BEGIN
   RETURN 0;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 -- Find the parent of an address with addr:street/addr:place tag.
@@ -299,7 +299,7 @@ BEGIN
   RETURN parent_place_id;
 END;
 $$
-LANGUAGE plpgsql STABLE;
+LANGUAGE plpgsql STABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION delete_location(OLD_place_id BIGINT)
@@ -337,7 +337,7 @@ BEGIN
                      ST_Project(geom::geography, radius, 3.9269908)::geometry));
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION add_location(place_id BIGINT, country_code varchar(2),
@@ -455,7 +455,7 @@ BEGIN
   RETURN;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION split_geometry(geometry GEOMETRY)
@@ -483,7 +483,7 @@ BEGIN
   RETURN;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 CREATE OR REPLACE FUNCTION simplify_large_polygons(geometry GEOMETRY)
   RETURNS GEOMETRY
@@ -497,7 +497,7 @@ BEGIN
   RETURN geometry;
 END;
 $$
-LANGUAGE plpgsql IMMUTABLE;
+LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION place_force_delete(placeid BIGINT)

lib-sql/tokenizer/icu_tokenizer.sql

@@ -12,7 +12,7 @@ CREATE OR REPLACE FUNCTION token_get_name_search_tokens(info JSONB)
   RETURNS INTEGER[]
 AS $$
   SELECT (info->>'names')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 -- Get tokens for matching the place name against others.
@@ -22,7 +22,7 @@ CREATE OR REPLACE FUNCTION token_get_name_match_tokens(info JSONB)
   RETURNS INTEGER[]
 AS $$
   SELECT (info->>'names')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 -- Return the housenumber tokens applicable for the place.
@@ -30,7 +30,7 @@ CREATE OR REPLACE FUNCTION token_get_housenumber_search_tokens(info JSONB)
   RETURNS INTEGER[]
 AS $$
   SELECT (info->>'hnr_tokens')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 -- Return the housenumber in the form that it can be matched during search.
@@ -38,77 +38,77 @@ CREATE OR REPLACE FUNCTION token_normalized_housenumber(info JSONB)
   RETURNS TEXT
 AS $$
   SELECT info->>'hnr';
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION token_is_street_address(info JSONB)
   RETURNS BOOLEAN
 AS $$
   SELECT info->>'street' is not null or info->>'place' is null;
-$$ LANGUAGE SQL IMMUTABLE;
+$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION token_has_addr_street(info JSONB)
   RETURNS BOOLEAN
 AS $$
   SELECT info->>'street' is not null and info->>'street' != '{}';
-$$ LANGUAGE SQL IMMUTABLE;
+$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION token_has_addr_place(info JSONB)
   RETURNS BOOLEAN
 AS $$
   SELECT info->>'place' is not null;
-$$ LANGUAGE SQL IMMUTABLE;
+$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION token_matches_street(info JSONB, street_tokens INTEGER[])
   RETURNS BOOLEAN
 AS $$
   SELECT (info->>'street')::INTEGER[] && street_tokens
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION token_matches_place(info JSONB, place_tokens INTEGER[])
   RETURNS BOOLEAN
 AS $$
   SELECT (info->>'place')::INTEGER[] <@ place_tokens
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION token_addr_place_search_tokens(info JSONB)
   RETURNS INTEGER[]
 AS $$
   SELECT (info->>'place')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION token_get_address_keys(info JSONB)
   RETURNS SETOF TEXT
 AS $$
   SELECT * FROM jsonb_object_keys(info->'addr');
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION token_get_address_search_tokens(info JSONB, key TEXT)
   RETURNS INTEGER[]
 AS $$
   SELECT (info->'addr'->>key)::INTEGER[];
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION token_matches_address(info JSONB, key TEXT, tokens INTEGER[])
   RETURNS BOOLEAN
 AS $$
   SELECT (info->'addr'->>key)::INTEGER[] <@ tokens;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 CREATE OR REPLACE FUNCTION token_get_postcode(info JSONB)
   RETURNS TEXT
 AS $$
   SELECT info->>'postcode';
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 
 -- Return token info that should be saved permanently in the database.
@@ -116,7 +116,7 @@ CREATE OR REPLACE FUNCTION token_strip_info(info JSONB)
   RETURNS JSONB
 AS $$
   SELECT NULL::JSONB;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
+$$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE;
 
 --------------- private functions ----------------------------------------------
 
@@ -128,16 +128,14 @@ DECLARE
   partial_terms TEXT[] = '{}'::TEXT[];
   term TEXT;
   term_id INTEGER;
-  term_count INTEGER;
 BEGIN
   SELECT min(word_id) INTO full_token
     FROM word WHERE word = norm_term and type = 'W';
 
   IF full_token IS NULL THEN
     full_token := nextval('seq_word');
-    INSERT INTO word (word_id, word_token, type, word, info)
-      SELECT full_token, lookup_term, 'W', norm_term,
-             json_build_object('count', 0)
+    INSERT INTO word (word_id, word_token, type, word)
+      SELECT full_token, lookup_term, 'W', norm_term
         FROM unnest(lookup_terms) as lookup_term;
   END IF;
 
@@ -150,14 +148,67 @@ BEGIN
 
   partial_tokens := '{}'::INT[];
   FOR term IN SELECT unnest(partial_terms) LOOP
-    SELECT min(word_id), max(info->>'count') INTO term_id, term_count
+    SELECT min(word_id) INTO term_id
       FROM word WHERE word_token = term and type = 'w';
 
     IF term_id IS NULL THEN
       term_id := nextval('seq_word');
-      term_count := 0;
-      INSERT INTO word (word_id, word_token, type, info)
-        VALUES (term_id, term, 'w', json_build_object('count', term_count));
+      INSERT INTO word (word_id, word_token, type)
+        VALUES (term_id, term, 'w');
+    END IF;
+
+    partial_tokens := array_merge(partial_tokens, ARRAY[term_id]);
+  END LOOP;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION getorcreate_full_word(norm_term TEXT,
+                                                 lookup_terms TEXT[],
+                                                 lookup_norm_terms TEXT[],
+                                                 OUT full_token INT,
+                                                 OUT partial_tokens INT[])
+  AS $$
+DECLARE
+  partial_terms TEXT[] = '{}'::TEXT[];
+  term TEXT;
+  term_id INTEGER;
+BEGIN
+  SELECT min(word_id) INTO full_token
+    FROM word WHERE word = norm_term and type = 'W';
+
+  IF full_token IS NULL THEN
+    full_token := nextval('seq_word');
+    IF lookup_norm_terms IS NULL THEN
+      INSERT INTO word (word_id, word_token, type, word)
+        SELECT full_token, lookup_term, 'W', norm_term
+          FROM unnest(lookup_terms) as lookup_term;
+    ELSE
+      INSERT INTO word (word_id, word_token, type, word, info)
+        SELECT full_token, t.lookup, 'W', norm_term,
+               CASE WHEN norm_term = t.norm THEN null
+               ELSE json_build_object('lookup', t.norm) END
+          FROM unnest(lookup_terms, lookup_norm_terms) as t(lookup, norm);
+    END IF;
+  END IF;
+
+  FOR term IN SELECT unnest(string_to_array(unnest(lookup_terms), ' ')) LOOP
+    term := trim(term);
+    IF NOT (ARRAY[term] <@ partial_terms) THEN
+      partial_terms := partial_terms || term;
+    END IF;
+  END LOOP;
+
+  partial_tokens := '{}'::INT[];
+  FOR term IN SELECT unnest(partial_terms) LOOP
+    SELECT min(word_id) INTO term_id
+      FROM word WHERE word_token = term and type = 'w';
+
+    IF term_id IS NULL THEN
+      term_id := nextval('seq_word');
+      INSERT INTO word (word_id, word_token, type)
+        VALUES (term_id, term, 'w');
     END IF;
 
     partial_tokens := array_merge(partial_tokens, ARRAY[term_id]);

nominatim-cli.py

@@ -3,7 +3,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Helper script for development to run nominatim from the source directory.
@@ -15,4 +15,4 @@ sys.path.insert(1, str((Path(__file__) / '..' / 'src').resolve()))
 
 from nominatim_db import cli
 
-exit(cli.nominatim(module_dir=None, osm2pgsql_path=None))
+exit(cli.nominatim())

packaging/nominatim-db/pyproject.toml

@@ -19,7 +19,6 @@ dependencies = [
     "python-dotenv",
     "jinja2",
     "pyYAML>=5.1",
-    "datrie",
     "psutil",
     "PyICU"
 ]

packaging/nominatim-db/scripts/nominatim

@@ -2,4 +2,4 @@
 
 from nominatim_db import cli
 
-exit(cli.nominatim(osm2pgsql_path=None))
+exit(cli.nominatim())

settings/address-levels.json

@@ -216,6 +216,14 @@
       }
   }
 },
+{ "countries" : ["sa"],
+  "tags" : {
+      "place" : {
+          "province" : 12,
+          "municipality" : 18
+      }
+  }
+},
 { "countries" : ["sk"],
   "tags" : {
       "boundary" : {

settings/country_settings.yaml

@@ -944,7 +944,7 @@ kp:
 # South Korea (대한민국)
 kr:
     partition: 49
-    languages: ko, en
+    languages: ko
     names: !include country-names/kr.yaml
     postcode:
       pattern: "ddddd"
@@ -1809,7 +1809,8 @@ us:
     languages: en
     names: !include country-names/us.yaml
     postcode:
-      pattern: "ddddd"
+      pattern: "(ddddd)(?:-dddd)?"
+      output: \1
 
 
 # Uruguay (Uruguay)

settings/env.defaults

@@ -192,6 +192,13 @@ NOMINATIM_REQUEST_TIMEOUT=60
 # to geocode" instead.
 NOMINATIM_SEARCH_WITHIN_COUNTRIES=False
 
+# Specifies the order in which different name tags are used.
+# The values in this list determine the preferred order of name variants,
+# including language-specific names.
+# Comma-separated list, where :XX stands for language-specific tags
+# (e.g. name:en) and no :XX stands for general tags (e.g. name).
+NOMINATIM_OUTPUT_NAMES=name:XX,name,brand,official_name:XX,short_name:XX,official_name,short_name,ref
+
 ### Log settings
 #
 # The following options allow to enable logging of API requests.

settings/icu-rules/variants-ca.yaml

@@ -4,7 +4,7 @@
     -  aparcament -> aparc
     -  apartament -> apmt
     -  apartat -> apt
-    - àtic -> àt
+    -  àtic -> àt
     -  autopista -> auto
     -  autopista -> autop
     -  autovia -> autov
@@ -19,7 +19,6 @@
     -  biblioteca -> bibl
     -  bloc -> bl
     -  carrer -> c
-    -  carrer -> c/
     -  carreró -> cró
     -  carretera -> ctra
     -  cantonada -> cant
@@ -58,7 +57,6 @@
     -  número -> n
     -  sense número -> s/n
     -  parada -> par
-    -  parcel·la -> parc
     -  passadís -> pdís
     -  passatge -> ptge
     -  passeig -> pg

settings/icu-rules/variants-en.yaml

@@ -1,438 +1,393 @@
 # Source: https://wiki.openstreetmap.org/wiki/Name_finder:Abbreviations#English
+# Source: https://pe.usps.com/text/pub28/28apc_002.htm
 - lang: en
   words:
     -  Access -> Accs
     -  Air Force Base -> AFB
     -  Air National Guard Base -> ANGB
     -  Airport -> Aprt
-    -  Alley -> Al
-    -  Alley -> All
-    -  Alley -> Ally
-    -  Alley -> Aly
+    -  Alley -> Al,All,Ally,Aly
     -  Alleyway -> Alwy
     -  Amble -> Ambl
+    -  Anex -> Anx
     -  Apartments -> Apts
-    -  Approach -> Apch
-    -  Approach -> App
+    -  Approach -> Apch,App
     -  Arcade -> Arc
     -  Arterial -> Artl
     -  Artery -> Arty
-    -  Avenue -> Av
-    -  Avenue -> Ave
+    -  Avenue -> Av,Ave
     -  Back -> Bk
     -  Banan -> Ba
-    -  Basin -> Basn
-    -  Basin -> Bsn
+    -  Basin -> Basn,Bsn
+    -  Bayou -> Byu
     -  Beach -> Bch
-    -  Bend -> Bend
     -  Bend -> Bnd
     -  Block -> Blk
+    -  Bluff -> Blf
+    -  Bluffs -> Blfs
     -  Boardwalk -> Bwlk
-    -  Boulevard -> Blvd
-    -  Boulevard -> Bvd
+    -  Bottom -> Btm
+    -  Boulevard -> Blvd,Bvd
     -  Boundary -> Bdy
     -  Bowl -> Bl
     -  Brace -> Br
     -  Brae -> Br
-    -  Brae -> Brae
+    -  Branch -> Br
     -  Break -> Brk
-    -  Bridge -> Bdge
-    -  Bridge -> Br
-    -  Bridge -> Brdg
-    -  Bridge -> Bri
-    -  Broadway -> Bdwy
-    -  Broadway -> Bway
-    -  Broadway -> Bwy
+    -  Bridge$ -> Bdge,Br,Brdg,Brg,Bri
+    -  Broadway -> Bdwy,Bway,Bwy
     -  Brook -> Brk
+    -  Brooks -> Brks
     -  Brow -> Brw
-    -  Brow -> Brow
-    -  Buildings -> Bldgs
-    -  Buildings -> Bldngs
+    -  Buildings -> Bldgs,Bldngs
     -  Business -> Bus
-    -  Bypass -> Bps
-    -  Bypass -> Byp
-    -  Bypass -> Bypa
+    -  Burg -> Bg
+    -  Burgs -> Bgs
+    -  Bypass -> Bps,Byp,Bypa
     -  Byway -> Bywy
+    -  Camp -> Cp
+    -  Canyon -> Cyn
+    -  Cape -> Cpe
     -  Caravan -> Cvn
-    -  Causeway -> Caus
-    -  Causeway -> Cswy
-    -  Causeway -> Cway
-    -  Center -> Cen
-    -  Center -> Ctr
+    -  Causeway -> Caus,Cswy,Cway
+    -  Center,Centre -> Cen,Ctr
+    -  Centers -> Ctrs
     -  Central -> Ctrl
-    -  Centre -> Cen
-    -  Centre -> Ctr
     -  Centreway -> Cnwy
     -  Chase -> Ch
     -  Church -> Ch
     -  Circle -> Cir
-    -  Circuit -> Cct
-    -  Circuit -> Ci
-    -  Circus -> Crc
-    -  Circus -> Crcs
+    -  Circles -> Cirs
+    -  Circuit -> Cct,Ci
+    -  Circus -> Crc,Crcs
     -  City -> Cty
+    -  Cliff -> Clf
+    -  Cliffs -> Clfs
     -  Close -> Cl
-    -  Common -> Cmn
-    -  Common -> Comm
+    -  Club -> Clb
+    -  Common -> Cmn,Comm
+    -  Commons -> Cmns
     -  Community -> Comm
     -  Concourse -> Cnc
     -  Concourse -> Con
     -  Copse -> Cps
-    -  Corner -> Cnr
-    -  Corner -> Crn
+    -  Corner -> Cor,Cnr,Crn
+    -  Corners -> Cors
     -  Corso -> Cso
     -  Cottages -> Cotts
     -  County -> Co
     -  County Road -> CR
     -  County Route -> CR
-    -  Court -> Crt
-    -  Court -> Ct
+    -  Course -> Crse
+    -  Court -> Crt,Ct
+    -  Courts -> Cts
     -  Courtyard -> Cyd
     -  Courtyard -> Ctyd
-    -  Cove -> Ce
-    -  Cove -> Cov
-    -  Cove -> Cove
-    -  Cove -> Cv
-    -  Creek -> Ck
-    -  Creek -> Cr
-    -  Creek -> Crk
+    -  Cove$ -> Ce,Cov,Cv
+    -  Coves -> Cvs
+    -  Creek$ -> Ck,Cr,Crk
     -  Crescent -> Cr
     -  Crescent -> Cres
-    -  Crest -> Crst
-    -  Crest -> Cst
+    -  Crest -> Crst,Cst
     -  Croft -> Cft
-    -  Cross -> Cs
-    -  Cross -> Crss
-    -  Crossing -> Crsg
-    -  Crossing -> Csg
-    -  Crossing -> Xing
-    -  Crossroad -> Crd
+    -  Cross -> Cs,Crss
+    -  Crossing -> Crsg,Csg,Xing
+    -  Crossroad -> Crd,Xrd
+    -  Crossroads -> Xrds
     -  Crossway -> Cowy
-    -  Cul-de-sac -> Cds
-    -  Cul-de-sac -> Csac
-    -  Curve -> Cve
+    -  Cul-de-sac -> Cds,Csac
+    -  Curve -> Cve,Curv
     -  Cutting -> Cutt
     -  Dale -> Dle
-    -  Dale -> Dale
+    -  Dam -> Dm
     -  Deviation -> Devn
-    -  Dip -> Dip
     -  Distributor -> Dstr
+    -  Divide -> Dv
     -  Down -> Dn
     -  Downs -> Dn
-    -  Drive -> Dr
-    -  Drive -> Drv
-    -  Drive -> Dv
+    -  Drive -> Dr,Drv,Dv
+    -  Drives -> Drs
     -  Drive-In => Drive-In # prevent abbreviation here
-    -  Driveway -> Drwy
-    -  Driveway -> Dvwy
-    -  Driveway -> Dwy
+    -  Driveway -> Drwy,Dvwy,Dwy
     -  East -> E
     -  Edge -> Edg
-    -  Edge -> Edge
     -  Elbow -> Elb
-    -  End -> End
     -  Entrance -> Ent
     -  Esplanade -> Esp
     -  Estate -> Est
-    -  Expressway -> Exp
-    -  Expressway -> Expy
-    -  Expressway -> Expwy
-    -  Expressway -> Xway
+    -  Estates -> Ests
+    -  Expressway -> Exp,Expy,Expwy,Xway
     -  Extension -> Ex
-    -  Fairway -> Fawy
-    -  Fairway -> Fy
+    -  Extensions -> Exts
+    -  Fairway -> Fawy,Fy
+    -  Falls -> Fls
     -  Father -> Fr
-    -  Ferry -> Fy
-    -  Field -> Fd
+    -  Ferry -> Fy,Fry
+    -  Field -> Fd,Fld
+    -  Fields -> Flds
     -  Fire Track -> Ftrk
     -  Firetrail -> Fit
-    -  Flat -> Fl
-    -  Flat -> Flat
+    -  Flat -> Fl,Flt
+    -  Flats -> Flts
     -  Follow -> Folw
     -  Footway -> Ftwy
+    -  Ford -> Frd
+    -  Fords -> Frds
     -  Foreshore -> Fshr
+    -  Forest -> Frst
     -  Forest Service Road -> FSR
+    -  Forge -> Frg
+    -  Forges -> Frgs
     -  Formation -> Form
+    -  Fork -> Frk
+    -  Forks -> Frks
     -  Fort -> Ft
-    -  Freeway -> Frwy
-    -  Freeway -> Fwy
+    -  Freeway -> Frwy,Fwy
     -  Front -> Frnt
-    -  Frontage -> Fr
-    -  Frontage -> Frtg
-    -  Gap -> Gap
+    -  Frontage -> Fr,Frtg
     -  Garden -> Gdn
-    -  Gardens -> Gdn
-    -  Gardens -> Gdns
-    -  Gate -> Ga
-    -  Gate -> Gte
-    -  Gates -> Ga
-    -  Gates -> Gte
-    -  Gateway -> Gwy
+    -  Gardens -> Gdn,Gdns
+    -  Gate,Gates -> Ga,Gte
+    -  Gateway -> Gwy,Gtwy
     -  George -> Geo
-    -  Glade -> Gl
-    -  Glade -> Gld
-    -  Glade -> Glde
+    -  Glade$ -> Gl,Gld,Glde
     -  Glen -> Gln
-    -  Glen -> Glen
+    -  Glens -> Glns
     -  Grange -> Gra
-    -  Green -> Gn
-    -  Green -> Grn
+    -  Green -> Gn,Grn
+    -  Greens -> Grns
     -  Ground -> Grnd
-    -  Grove -> Gr
-    -  Grove -> Gro
+    -  Grove$ -> Gr,Gro,Grv
+    -  Groves -> Grvs
     -  Grovet -> Gr
     -  Gully -> Gly
-    -  Harbor -> Hbr
-    -  Harbour -> Hbr
+    -  Harbor -> Hbr,Harbour
+    -  Harbors -> Hbrs
+    -  Harbour -> Hbr,Harbor
     -  Haven -> Hvn
     -  Head -> Hd
     -  Heads -> Hd
-    -  Heights -> Hgts
-    -  Heights -> Ht
-    -  Heights -> Hts
+    -  Heights -> Hgts,Ht,Hts
     -  High School -> HS
-    -  Highroad -> Hird
-    -  Highroad -> Hrd
+    -  Highroad -> Hird,Hrd
     -  Highway -> Hwy
-    -  Hill -> Hill
     -  Hill -> Hl
-    -  Hills -> Hl
-    -  Hills -> Hls
+    -  Hills -> Hl,Hls
+    -  Hollow -> Holw
     -  Hospital -> Hosp
-    -  House -> Ho
-    -  House -> Hse
+    -  House -> Ho,Hse
     -  Industrial -> Ind
+    -  Inlet -> Inlt
     -  Interchange -> Intg
     -  International -> Intl
-    -  Island -> I
-    -  Island -> Is
-    -  Junction -> Jctn
-    -  Junction -> Jnc
+    -  Island -> I,Is
+    -  Islands -> Iss
+    -  Junction -> Jct,Jctn,Jnc
+    -  Junctions -> Jcts
     -  Junior -> Jr
-    -  Key -> Key
+    -  Key -> Ky
+    -  Keys -> Kys
+    -  Knoll -> Knl
+    -  Knolls -> Knls
     -  Lagoon -> Lgn
-    -  Lakes -> L
-    -  Landing -> Ldg
-    -  Lane -> La
-    -  Lane -> Lane
-    -  Lane -> Ln
+    -  Lake -> Lk
+    -  Lakes -> L,Lks
+    -  Landing -> Ldg,Lndg
+    -  Lane -> La,Ln
     -  Laneway -> Lnwy
-    -  Line -> Line
+    -  Light -> Lgt
+    -  Lights -> Lgts
     -  Line -> Ln
-    -  Link -> Link
     -  Link -> Lk
-    -  Little -> Lit
-    -  Little -> Lt
+    -  Little -> Lit,Lt
+    -  Loaf -> Lf
+    -  Lock -> Lck
+    -  Locks -> Lcks
     -  Lodge -> Ldg
     -  Lookout -> Lkt
-    -  Loop -> Loop
     -  Loop -> Lp
-    -  Lower -> Low
-    -  Lower -> Lr
-    -  Lower -> Lwr
-    -  Mall -> Mall
+    -  Lower -> Low,Lr,Lwr
     -  Mall -> Ml
     -  Manor -> Mnr
+    -  Manors -> Mnrs
     -  Mansions -> Mans
     -  Market -> Mkt
     -  Meadow -> Mdw
-    -  Meadows -> Mdw
-    -  Meadows -> Mdws
+    -  Meadows -> Mdw,Mdws
     -  Mead -> Md
-    -  Meander -> Mdr
-    -  Meander -> Mndr
-    -  Meander -> Mr
+    -  Meander -> Mdr,Mndr,Mr
     -  Medical -> Med
     -  Memorial -> Mem
-    -  Mews -> Mews
     -  Mews -> Mw
     -  Middle -> Mid
     -  Middle School -> MS
     -  Mile -> Mi
     -  Military -> Mil
-    -  Motorway -> Mtwy
-    -  Motorway -> Mwy
+    -  Mill -> Ml
+    -  Mills -> Mls
+    -  Mission -> Msn
+    -  Motorway -> Mtwy,Mwy
     -  Mount -> Mt
     -  Mountain -> Mtn
-    -  Mountains -> Mtn
+    -  Mountains$ -> Mtn,Mtns
     -  Municipal -> Mun
     -  Museum -> Mus
     -  National Park -> NP
     -  National Recreation Area -> NRA
     -  National Wildlife Refuge Area -> NWRA
+    -  Neck -> Nck
     -  Nook -> Nk
-    -  Nook -> Nook
     -  North -> N
     -  Northeast -> NE
     -  Northwest -> NW
-    -  Outlook -> Out
-    -  Outlook -> Otlk
+    -  Orchard -> Orch
+    -  Outlook -> Out,Otlk
+    -  Overpass -> Opas
     -  Parade -> Pde
     -  Paradise -> Pdse
-    -  Park -> Park
     -  Park -> Pk
     -  Parklands -> Pkld
-    -  Parkway -> Pkwy
-    -  Parkway -> Pky
-    -  Parkway -> Pwy
-    -  Pass -> Pass
+    -  Parkway -> Pkwy,Pky,Pwy
+    -  Parkways -> Pkwy
     -  Pass -> Ps
     -  Passage -> Psge
-    -  Path -> Path
-    -  Pathway -> Phwy
-    -  Pathway -> Pway
-    -  Pathway -> Pwy
+    -  Pathway -> Phwy,Pway,Pwy
     -  Piazza -> Piaz
     -  Pike -> Pk
+    -  Pine -> Pne
+    -  Pines -> Pnes
     -  Place -> Pl
-    -  Plain -> Pl
-    -  Plains -> Pl
+    -  Plain -> Pl,Pln
+    -  Plains -> Pl,Plns
     -  Plateau -> Plat
-    -  Plaza -> Pl
-    -  Plaza -> Plz
-    -  Plaza -> Plza
+    -  Plaza -> Pl,Plz,Plza
     -  Pocket -> Pkt
-    -  Point -> Pnt
-    -  Point -> Pt
-    -  Port -> Port
-    -  Port -> Pt
+    -  Point -> Pnt,Pt
+    -  Points -> Pts
+    -  Port -> Prt,Pt
+    -  Ports -> Prts
     -  Post Office -> PO
+    -  Prairie -> Pr
     -  Precinct -> Pct
-    -  Promenade -> Prm
-    -  Promenade -> Prom
-    -  Quad -> Quad
+    -  Promenade -> Prm,Prom
     -  Quadrangle -> Qdgl
-    -  Quadrant -> Qdrt
-    -  Quadrant -> Qd
+    -  Quadrant -> Qdrt,Qd
     -  Quay -> Qy
     -  Quays -> Qy
     -  Quays -> Qys
+    -  Radial -> Radl
     -  Ramble -> Ra
     -  Ramble -> Rmbl
-    -  Range -> Rge
-    -  Range -> Rnge
+    -  Ranch -> Rnch
+    -  Range -> Rge,Rnge
+    -  Rapid -> Rpd
+    -  Rapids -> Rpds
     -  Reach -> Rch
     -  Reservation -> Res
     -  Reserve -> Res
     -  Reservoir -> Res
-    -  Rest -> Rest
     -  Rest -> Rst
-    -  Retreat -> Rt
-    -  Retreat -> Rtt
+    -  Retreat -> Rt,Rtt
     -  Return -> Rtn
-    -  Ridge -> Rdg
-    -  Ridge -> Rdge
+    -  Ridge -> Rdg,Rdge
+    -  Ridges -> Rdgs
     -  Ridgeway -> Rgwy
     -  Right of Way -> Rowy
     -  Rise -> Ri
-    -  Rise -> Rise
-    -  River -> R
-    -  River -> Riv
-    -  River -> Rvr
+    -  ^River -> R,Riv,Rvr
+    -  River$ -> R,Riv,Rvr
     -  Riverway -> Rvwy
     -  Riviera -> Rvra
     -  Road -> Rd
     -  Roads -> Rds
     -  Roadside -> Rdsd
-    -  Roadway -> Rdwy
-    -  Roadway -> Rdy
-    -  Robert -> Robt
+    -  Roadway -> Rdwy,Rdy
     -  Rocks -> Rks
     -  Ronde -> Rnde
     -  Rosebowl -> Rsbl
     -  Rotary -> Rty
     -  Round -> Rnd
-    -  Route -> Rt
-    -  Route -> Rte
-    -  Row -> Row
-    -  Rue -> Rue
-    -  Run -> Run
+    -  Route -> Rt,Rte
     -  Saint -> St
     -  Saints -> SS
     -  Senior -> Sr
-    -  Serviceway -> Swy
-    -  Serviceway -> Svwy
+    -  Serviceway -> Swy,Svwy
+    -  Shoal -> Shl
+    -  Shore -> Shr
+    -  Shores -> Shrs
     -  Shunt -> Shun
     -  Siding -> Sdng
     -  Sister -> Sr
+    -  Skyway -> Skwy
     -  Slope -> Slpe
     -  Sound -> Snd
-    -  South -> S
-    -  South -> Sth
+    -  South -> S,Sth
     -  Southeast -> SE
     -  Southwest -> SW
-    -  Spur -> Spur
+    -  Spring -> Spg
+    -  Springs -> Spgs
+    -  Spurs -> Spur
     -  Square -> Sq
+    -  Squares -> Sqs
     -  Stairway -> Strwy
-    -  State Highway -> SH
-    -  State Highway -> SHwy
+    -  State Highway -> SH,SHwy
     -  State Route -> SR
-    -  Station -> Sta
-    -  Station -> Stn
-    -  Strand -> Sd
-    -  Strand -> Stra
+    -  Station -> Sta,Stn
+    -  Strand -> Sd,Stra
+    -  Stravenue -> Stra
+    -  Stream -> Strm
     -  Street -> St
+    -  Streets -> Sts
     -  Strip -> Strp
     -  Subway -> Sbwy
+    -  Summit -> Smt
     -  Tarn -> Tn
-    -  Tarn -> Tarn
     -  Terminal -> Term
-    -  Terrace -> Tce
-    -  Terrace -> Ter
-    -  Terrace -> Terr
-    -  Thoroughfare -> Thfr
-    -  Thoroughfare -> Thor
-    -  Tollway -> Tlwy
-    -  Tollway -> Twy
-    -  Top -> Top
-    -  Tor -> Tor
+    -  Terrace -> Tce,Ter,Terr
+    -  Thoroughfare -> Thfr,Thor
+    -  Throughway -> Trwy
+    -  Tollway -> Tlwy,Twy
     -  Towers -> Twrs
     -  Township -> Twp
     -  Trace -> Trce
-    -  Track -> Tr
-    -  Track -> Trk
+    -  Track -> Tr,Trak,Trk
+    -  Trafficway -> Trfy
     -  Trail -> Trl
     -  Trailer -> Trlr
     -  Triangle -> Tri
     -  Trunkway -> Tkwy
-    -  Tunnel -> Tun
-    -  Turn -> Tn
-    -  Turn -> Trn
-    -  Turn -> Turn
-    -  Turnpike -> Tpk
-    -  Turnpike -> Tpke
-    -  Underpass -> Upas
-    -  Underpass -> Ups
-    -  University -> Uni
-    -  University -> Univ
+    -  Tunnel -> Tun,Tunl
+    -  Turn -> Tn,Trn
+    -  Turnpike -> Tpk,Tpke
+    -  Underpass -> Upas,Ups
+    -  Union -> Un
+    -  Unions -> Uns
+    -  University -> Uni,Univ
     -  Upper -> Up
     -  Upper -> Upr
     -  Vale -> Va
-    -  Vale -> Vale
+    -  Valley -> Vly
     -  Valley -> Vy
-    -  Viaduct -> Vdct
-    -  Viaduct -> Via
-    -  Viaduct -> Viad
+    -  Valleys -> Vlys
+    -  Viaduct$ -> Vdct,Via,Viad
     -  View -> Vw
-    -  View -> View
-    -  Village -> Vill
+    -  Views -> Vws
+    -  Village -> Vill,Vlg
+    -  Villages -> Vlgs
     -  Villas -> Vlls
-    -  Vista -> Vst
-    -  Vista -> Vsta
-    -  Walk -> Walk
-    -  Walk -> Wk
-    -  Walk -> Wlk
-    -  Walkway -> Wkwy
-    -  Walkway -> Wky
+    -  Ville -> Vl
+    -  Vista -> Vis,Vst,Vsta
+    -  Walk -> Wk,Wlk
+    -  Walks -> Walk
+    -  Walkway -> Wkwy,Wky
     -  Waters -> Wtr
-    -  Way -> Way
     -  Way -> Wy
+    -  Well -> Wl
+    -  Wells -> Wls
     -  West -> W
     -  Wharf -> Whrf
     -  William -> Wm
     -  Wynd -> Wyn
-    -  Wynd -> Wynd
-    -  Yard -> Yard
     -  Yard -> Yd
 - lang: en
   country: ca

settings/icu-rules/variants-es.yaml

@@ -30,7 +30,6 @@
     -  Bloque -> Blq
     -  Bulevar -> Blvr
     -  Boulevard -> Blvd
-    -  Calle -> C/
     -  Calle -> C
     -  Calle -> Cl
     -  Calleja -> Cllja

settings/icu-rules/variants-fr.yaml

@@ -3,20 +3,16 @@
   words:
     -  Abbaye -> ABE
     -  Agglomération -> AGL
-    -  Aire -> AIRE
     -  Aires -> AIRE
     -  Allée -> ALL
-    -  Allée -> All
     -  Allées -> ALL
     -  Ancien chemin -> ACH
     -  Ancienne route -> ART
     -  Anciennes routes -> ART
-    -  Anse -> ANSE
     -  Arcade -> ARC
     -  Arcades -> ARC
     -  Autoroute -> AUT
     -  Avenue -> AV
-    -  Avenue -> Av
     -  Barrière -> BRE
     -  Barrières -> BRE
     -  Bas chemin -> BCH
@@ -28,16 +24,11 @@
     -  Berges -> BER
     -  Bois -> BOIS
     -  Boucle -> BCLE
-    -  Boulevard -> Bd
     -  Boulevard -> BD
     -  Bourg -> BRG
     -  Butte -> BUT
-    -  Cité -> CITE
     -  Cités -> CITE
-    -  Côte -> COTE
     -  Côteau -> COTE
-    -  Cale -> CALE
-    -  Camp -> CAMP
     -  Campagne -> CGNE
     -  Camping -> CPG
     -  Carreau -> CAU
@@ -56,17 +47,13 @@
     -  Chaussées -> CHS
     -  Chemin -> Ch
     -  Chemin -> CHE
-    -  Chemin -> Che
     -  Chemin vicinal -> CHV
     -  Cheminement -> CHEM
     -  Cheminements -> CHEM
     -  Chemins -> CHE
     -  Chemins vicinaux -> CHV
-    -  Chez -> CHEZ
     -  Château -> CHT
     -  Cloître -> CLOI
-    -  Clos -> CLOS
-    -  Col -> COL
     -  Colline -> COLI
     -  Collines -> COLI
     -  Contour -> CTR
@@ -74,9 +61,7 @@
     -  Corniches -> COR
     -  Cottage -> COTT
     -  Cottages -> COTT
-    -  Cour -> COUR
     -  Cours -> CRS
-    -  Cours -> Crs
     -  Darse -> DARS
     -  Degré -> DEG
     -  Degrés -> DEG
@@ -87,11 +72,8 @@
     -  Domaine -> DOM
     -  Domaines -> DOM
     -  Écluse -> ECL
-    -  Écluse -> ÉCL
     -  Écluses -> ECL
-    -  Écluses -> ÉCL
     -  Église -> EGL
-    -  Église -> ÉGL
     -  Enceinte -> EN
     -  Enclave -> ENV
     -  Enclos -> ENC
@@ -100,21 +82,16 @@
     -  Espace -> ESPA
     -  Esplanade -> ESP
     -  Esplanades -> ESP
-    -  Étang -> ETANG
-    -  Étang -> ÉTANG
     -  Faubourg -> FG
-    -  Faubourg -> Fg
     -  Ferme -> FRM
     -  Fermes -> FRM
     -  Fontaine -> FON
-    -  Fort -> FORT
     -  Forum -> FORM
     -  Fosse -> FOS
     -  Fosses -> FOS
     -  Foyer -> FOYR
     -  Galerie -> GAL
     -  Galeries -> GAL
-    -  Gare -> GARE
     -  Garenne -> GARN
     -  Grand boulevard -> GBD
     -  Grand ensemble -> GDEN
@@ -134,13 +111,9 @@
     -  Haut chemin -> HCH
     -  Hauts chemins -> HCH
     -  Hippodrome -> HIP
-    -  HLM -> HLM
-    -  Île -> ILE
-    -  Île -> ÎLE
     -  Immeuble -> IMM
     -  Immeubles -> IMM
     -  Impasse -> IMP
-    -  Impasse -> Imp
     -  Impasses -> IMP
     -  Jardin -> JARD
     -  Jardins -> JARD
@@ -150,13 +123,11 @@
     -  Lieu-dit -> LD
     -  Lotissement -> LOT
     -  Lotissements -> LOT
-    -  Mail -> MAIL
     -  Maison forestière -> MF
     -  Manoir -> MAN
     -  Marche -> MAR
     -  Marches -> MAR
     -  Maréchal -> MAL
-    -  Mas -> MAS
     -  Monseigneur -> Mgr
     -  Mont -> Mt
     -  Montée -> MTE
@@ -168,13 +139,9 @@
     -  Métro -> MÉT
     -  Nouvelle route -> NTE
     -  Palais -> PAL
-    -  Parc -> PARC
-    -  Parcs -> PARC
     -  Parking -> PKG
     -  Parvis -> PRV
     -  Passage -> PAS
-    -  Passage -> Pas
-    -  Passage -> Pass
     -  Passage à niveau -> PN
     -  Passe -> PASS
     -  Passerelle -> PLE
@@ -191,19 +158,14 @@
     -  Petite rue -> PTR
     -  Petites allées -> PTA
     -  Place -> PL
-    -  Place -> Pl
     -  Placis -> PLCI
     -  Plage -> PLAG
     -  Plages -> PLAG
     -  Plaine -> PLN
-    -  Plan -> PLAN
     -  Plateau -> PLT
     -  Plateaux -> PLT
     -  Pointe -> PNT
-    -  Pont -> PONT
-    -  Ponts -> PONT
     -  Porche -> PCH
-    -  Port -> PORT
     -  Porte -> PTE
     -  Portique -> PORQ
     -  Portiques -> PORQ
@@ -211,25 +173,19 @@
     -  Pourtour -> POUR
     -  Presqu’île -> PRQ
     -  Promenade -> PROM
-    -  Promenade -> Prom
-    -  Pré -> PRE
-    -  Pré -> PRÉ
     -  Périphérique -> PERI
     -  Péristyle -> PSTY
     -  Quai -> QU
-    -  Quai -> Qu
     -  Quartier -> QUA
     -  Raccourci -> RAC
     -  Raidillon -> RAID
     -  Rampe -> RPE
     -  Rempart -> REM
-    -  Roc -> ROC
     -  Rocade -> ROC
     -  Rond point -> RPT
     -  Roquet -> ROQT
     -  Rotonde -> RTD
     -  Route -> RTE
-    -  Route -> Rte
     -  Routes -> RTE
     -  Rue -> R
     -  Rue -> R
@@ -245,7 +201,6 @@
     -  Sentier -> SEN
     -  Sentiers -> SEN
     -  Square -> SQ
-    -  Square -> Sq
     -  Stade -> STDE
     -  Station -> STA
     -  Terrain -> TRN
@@ -254,13 +209,11 @@
     -  Terre plein -> TPL
     -  Tertre -> TRT
     -  Tertres -> TRT
-    -  Tour -> TOUR
     -  Traverse -> TRA
     -  Vallon -> VAL
     -  Vallée -> VAL
     -  Venelle -> VEN
     -  Venelles -> VEN
-    -  Via -> VIA
     -  Vieille route -> VTE
     -  Vieux chemin -> VCHE
     -  Villa -> VLA
@@ -269,7 +222,6 @@
     -  Villas -> VLA
     -  Voie -> VOI
     -  Voies -> VOI
-    -  Zone -> ZONE
     -  Zone artisanale -> ZA
     -  Zone d'aménagement concerté -> ZAC
     -  Zone d'aménagement différé -> ZAD
@@ -289,7 +241,6 @@
     -  Esplanade -> ESPL
     -  Passage -> PASS
     -  Plateau -> PLAT
-    -  Rang -> RANG
     -  Rond-point -> RDPT
     -  Sentier -> SENT
     -  Subdivision -> SUBDIV

settings/icu-rules/variants-it.yaml

@@ -29,7 +29,6 @@
     -  Prima -> I
     -  Primo -> I
     -  Primo -> 1
-    -  Primo -> 1°
     -  Quarta -> IV
     -  Quarto -> IV
     -  Quattro -> IV

settings/icu-rules/variants-no.yaml

@@ -1,11 +1,10 @@
 # Source: https://wiki.openstreetmap.org/wiki/Name_finder:Abbreviations#Norsk_-_Norwegian
-- lang: no
+- lang: "no"
   words:
     # convert between Nynorsk and Bookmal here
-    -  vei, veg => v,vn,vei,veg
-    -  veien, vegen -> v,vn,veien,vegen
-    -  gate -> g,gt
+    -  ~vei, ~veg -> v,vei,veg
+    -  ~veien, ~vegen -> vn,veien,vegen
     # convert between the two female forms
-    -  gaten, gata => g,gt,gaten,gata
+    -  gate, gaten, gata -> g,gt
     -  plass, plassen -> pl
     -  sving, svingen -> sv

settings/icu-rules/variants-ru.yaml

@@ -1,14 +1,128 @@
 # Source: https://wiki.openstreetmap.org/wiki/Name_finder:Abbreviations#.D0.A0.D1.83.D1.81.D1.81.D0.BA.D0.B8.D0.B9_-_Russian
+# Source: https://www.plantarium.ru/page/help/topic/abbreviations.html
+# Source: https://dic.academic.ru/dic.nsf/ruwiki/1871310
 - lang: ru
   words:
+    -  Академик, Академика -> Ак
+    -  акционерное общество -> АО
     -  аллея -> ал
+    -  архипелаг -> арх
+    -  атомная электростанция -> АЭС
+    -  аэродром -> аэрд
+    -  аэропорт -> аэрп
+    -  Башкирский, Башкирская, Башкирское, Башкирские -> Баш, Башк, Башкир
+    -  Белый, Белая, Белое. Белые -> Бел
+    -  болото -> бол
+    -  больница -> больн
+    -  Большой, Большая, Большое, Большие -> Б, Бол
+    -  брод -> бр
     -  бульвар -> бул
+    -  бухта -> бух
+    -  бывший, бывшая, бывшее, бывшие -> бывш
+    -  Великий, Великая, Великое, Великие -> Вел
+    -  Верхний, Верхняя, Верхнее, Верхние -> В, Верх
+    -  водокачка -> вдкч
+    -  водопад -> вдп
+    -  водохранилище -> вдхр
+    -  вокзал -> вкз, вокз
+    -  Восточный, Восточная, Восточное, Восточные -> В, Вост
+    -  вулкан -> влк
+    -  гидроэлектростанция -> ГЭС
+    -  гора -> г
+    -  город -> г
+    -  дворец культуры, дом культуры -> ДК
+    -  дворец спорта -> ДС
+    -  деревня -> д, дер
+    -  детский оздоровительный лагерь -> ДОЛ
+    -  дом -> д
+    -  дом отдыха -> Д О
+    -  железная дорога -> ж д
+    -  железнодорожный, железнодорожная, железнодорожное -> ж-д
+    -  железобетонных изделий -> ЖБИ
+    -  жилой комплекс -> ЖК
+    -  завод -> з-д
+    -  закрытое административно-территориальное образование -> ЗАТО
+    -  залив -> зал
+    -  Западный, Западная, Западное, Западные -> З, Зап, Запад
+    -  заповедник -> запов
+    -  имени -> им
+    -  институт -> инст
+    -  исправительная колония -> ИК
+    -  километр -> км
+    -  Красный, Красная, Красное, Красные -> Кр, Крас
+    -  лагерь -> лаг
+    -  Левый, Левая,Левое, Левые -> Л, Лев
+    -  ледник -> ледн
+    -  лесничество -> леснич
+    -  лесной, лесная, лесное -> лес
+    -  линия электропередачи -> ЛЭП
+    -  Малый, Малая, Малое, Малые -> М, Мал
+    -  Мордовский, Мордовская, Мордовское, Мордовские -> Мордов
+    -  морской, морская, морское -> мор
+    -  Московский, Московская, Московское, Московские -> Мос, Моск
+    -  мыс -> м
     -  набережная -> наб
+    -  Нижний, Нижняя, Нижнее, Нижние -> Ниж, Н
+    -  Новый, Новая, Новое, Новые -> Нов, Н
+    -  обгонный пункт -> обг п
+    -  область -> обл
+    -  озеро -> оз
+    -  особо охраняемая природная территория -> ООПТ
+    -  остановочный пункт -> о п
+    -  остров -> о
+    -  острова -> о-ва
+    -  парк культуры и отдыха -> ПКиО
+    -  перевал -> пер
     -  переулок -> пер
+    -  пещера -> пещ
+    -  пионерский лагерь -> пионерлаг
+    -  платформа -> пл, платф
     -  площадь -> пл
+    -  подсобное хозяйство -> подсоб хоз
+    -  полуостров -> п-ов
+    -  посёлок -> пос, п
+    -  посёлок городского типа -> п г т, пгт
+    -  Правый, Правая, Правое, Правые -> П, Пр, Прав
     -  проезд -> пр
     -  проспект -> просп
-    -  шоссе -> ш
+    -  пруд -> пр
+    -  пустыня -> пуст
+    -  разъезд -> рзд
+    -  район -> р-н
+    -  резинотехнических изделий -> РТИ
+    -  река -> р
+    -  речной, речная, речное -> реч, речн
+    -  Российский, Российская, Российское, Российские -> Рос
+    -  Русский, Русская, Русское, Русские -> Рус, Русск
+    -  ручей -> руч
+    -  садовое некоммерческое товарищество -> СНТ
+    -  садовые участки -> сад уч
+    -  санаторий -> сан
+    -  сарай -> сар
+    -  Северный, Северная, Северное, Северные -> С, Сев
+    -  село -> с
+    -  Сибирский, Сибирская, Сибирское, Сибирские -> Сиб
+    -  Советский, Советская, Советское, Советские -> Сов
+    -  совхоз -> свх
+    -  Сортировочный, Сортировочная, Сортировочное, Сортировочные -> Сорт
+    -  станция -> ст
+    -  Старый, Старая, Среднее, Средние -> Ср
+    -  Татарский, Татарская, Татарское, Татарские -> Тат, Татар
+    -  теплоэлекстростанция -> ТЭС
+    -  теплоэлектроцентраль -> ТЭЦ
+    -  техникум -> техн
+    -  тоннель, туннель -> тун
     -  тупик -> туп
     -  улица -> ул
-    -  область -> обл
+    -  Уральский, Уральская, Уральское, Уральские -> Ур, Урал
+    -  урочище -> ур
+    -  хозяйство -> хоз, хоз-во
+    -  хребет -> хр
+    -  хутор -> хут
+    -  Чёрный, Чёрная, Чёрное, Чёрные -> Черн
+    -  Чувашский, Чувашская, Чувашское, Чувашские -> Чуваш
+    -  шахта -> шах
+    -  школа -> шк
+    -  шоссе -> ш
+    -  элеватор -> элев
+    -  Южный, Южная, Южное, Южные -> Ю, Юж, Южн

settings/icu_tokenizer.yaml

@@ -46,7 +46,7 @@ sanitizers:
     - step: strip-brace-terms
     - step: tag-analyzer-by-language
       filter-kind: [".*name.*"]
-      whitelist: [bg,ca,cs,da,de,el,en,es,et,eu,fi,fr,gl,hu,it,ja,mg,ms,nl,no,pl,pt,ro,ru,sk,sl,sv,tr,uk,vi]
+      whitelist: [bg,ca,cs,da,de,el,en,es,et,eu,fi,fr,gl,hu,it,ja,mg,ms,nl,"no",pl,pt,ro,ru,sk,sl,sv,tr,uk,vi]
       use-defaults: all
       mode: append
     - step: tag-japanese
@@ -158,7 +158,7 @@ token-analysis:
       mode: variant-only
       variants:
           - !include icu-rules/variants-nl.yaml
-    - id: no
+    - id: "no"
       analyzer: generic
       mode: variant-only
       variants:

src/nominatim_api/core.py

@@ -26,7 +26,7 @@ from .connection import SearchConnection
 from .status import get_status, StatusResult
 from .lookup import get_places, get_detailed_place
 from .reverse import ReverseGeocoder
-from .search import ForwardGeocoder, Phrase, PhraseType, make_query_analyzer
+from . import search as nsearch
 from . import types as ntyp
 from .results import DetailedResult, ReverseResult, SearchResults
 
@@ -207,7 +207,7 @@ class NominatimAPIAsync:
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
             if details.keywords:
-                await make_query_analyzer(conn)
+                await nsearch.make_query_analyzer(conn)
             return await get_detailed_place(conn, place, details)
 
     async def lookup(self, places: Sequence[ntyp.PlaceRef], **params: Any) -> SearchResults:
@@ -219,7 +219,7 @@ class NominatimAPIAsync:
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
             if details.keywords:
-                await make_query_analyzer(conn)
+                await nsearch.make_query_analyzer(conn)
             return await get_places(conn, places, details)
 
     async def reverse(self, coord: ntyp.AnyPoint, **params: Any) -> Optional[ReverseResult]:
@@ -237,7 +237,7 @@ class NominatimAPIAsync:
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
             if details.keywords:
-                await make_query_analyzer(conn)
+                await nsearch.make_query_analyzer(conn)
             geocoder = ReverseGeocoder(conn, details,
                                        self.reverse_restrict_to_country_area)
             return await geocoder.lookup(coord)
@@ -251,10 +251,10 @@ class NominatimAPIAsync:
 
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
-            geocoder = ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params),
-                                       self.config.get_int('REQUEST_TIMEOUT')
-                                       if self.config.REQUEST_TIMEOUT else None)
-            phrases = [Phrase(PhraseType.NONE, p.strip()) for p in query.split(',')]
+            geocoder = nsearch.ForwardGeocoder(conn, ntyp.SearchDetails.from_kwargs(params),
+                                               self.config.get_int('REQUEST_TIMEOUT')
+                                               if self.config.REQUEST_TIMEOUT else None)
+            phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p.strip()) for p in query.split(',')]
             return await geocoder.lookup(phrases)
 
     async def search_address(self, amenity: Optional[str] = None,
@@ -271,22 +271,22 @@ class NominatimAPIAsync:
             conn.set_query_timeout(self.query_timeout)
             details = ntyp.SearchDetails.from_kwargs(params)
 
-            phrases: List[Phrase] = []
+            phrases: List[nsearch.Phrase] = []
 
             if amenity:
-                phrases.append(Phrase(PhraseType.AMENITY, amenity))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_AMENITY, amenity))
             if street:
-                phrases.append(Phrase(PhraseType.STREET, street))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_STREET, street))
             if city:
-                phrases.append(Phrase(PhraseType.CITY, city))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_CITY, city))
             if county:
-                phrases.append(Phrase(PhraseType.COUNTY, county))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_COUNTY, county))
             if state:
-                phrases.append(Phrase(PhraseType.STATE, state))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_STATE, state))
             if postalcode:
-                phrases.append(Phrase(PhraseType.POSTCODE, postalcode))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_POSTCODE, postalcode))
             if country:
-                phrases.append(Phrase(PhraseType.COUNTRY, country))
+                phrases.append(nsearch.Phrase(nsearch.PHRASE_COUNTRY, country))
 
             if not phrases:
                 raise UsageError('Nothing to search for.')
@@ -304,14 +304,14 @@ class NominatimAPIAsync:
             else:
                 details.restrict_min_max_rank(4, 4)
 
-            if 'layers' not in params:
+            if details.layers is None:
                 details.layers = ntyp.DataLayer.ADDRESS
                 if amenity:
                     details.layers |= ntyp.DataLayer.POI
 
-            geocoder = ForwardGeocoder(conn, details,
-                                       self.config.get_int('REQUEST_TIMEOUT')
-                                       if self.config.REQUEST_TIMEOUT else None)
+            geocoder = nsearch.ForwardGeocoder(conn, details,
+                                               self.config.get_int('REQUEST_TIMEOUT')
+                                               if self.config.REQUEST_TIMEOUT else None)
             return await geocoder.lookup(phrases)
 
     async def search_category(self, categories: List[Tuple[str, str]],
@@ -328,15 +328,15 @@ class NominatimAPIAsync:
         async with self.begin() as conn:
             conn.set_query_timeout(self.query_timeout)
             if near_query:
-                phrases = [Phrase(PhraseType.NONE, p) for p in near_query.split(',')]
+                phrases = [nsearch.Phrase(nsearch.PHRASE_ANY, p) for p in near_query.split(',')]
             else:
                 phrases = []
                 if details.keywords:
-                    await make_query_analyzer(conn)
+                    await nsearch.make_query_analyzer(conn)
 
-            geocoder = ForwardGeocoder(conn, details,
-                                       self.config.get_int('REQUEST_TIMEOUT')
-                                       if self.config.REQUEST_TIMEOUT else None)
+            geocoder = nsearch.ForwardGeocoder(conn, details,
+                                               self.config.get_int('REQUEST_TIMEOUT')
+                                               if self.config.REQUEST_TIMEOUT else None)
             return await geocoder.lookup_pois(categories, phrases)
 
 

src/nominatim_api/localization.py

@@ -8,6 +8,7 @@
 Helper functions for localizing names of results.
 """
 from typing import Mapping, List, Optional
+from .config import Configuration
 
 import re
 
@@ -20,14 +21,18 @@ class Locales:
     """
 
     def __init__(self, langs: Optional[List[str]] = None):
+        self.config = Configuration(None)
         self.languages = langs or []
         self.name_tags: List[str] = []
 
-        # Build the list of supported tags. It is currently hard-coded.
-        self._add_lang_tags('name')
-        self._add_tags('name', 'brand')
-        self._add_lang_tags('official_name', 'short_name')
-        self._add_tags('official_name', 'short_name', 'ref')
+        parts = self.config.OUTPUT_NAMES.split(',')
+
+        for part in parts:
+            part = part.strip()
+            if part.endswith(":XX"):
+                self._add_lang_tags(part[:-3])
+            else:
+                self._add_tags(part)
 
     def __bool__(self) -> bool:
         return len(self.languages) > 0

src/nominatim_api/logging.py

@@ -342,7 +342,8 @@ HTML_HEADER: str = """<!DOCTYPE html>
   <title>Nominatim - Debug</title>
   <style>
 """ + \
-    (HtmlFormatter(nobackground=True).get_style_defs('.highlight') if CODE_HIGHLIGHT else '') + \
+    (HtmlFormatter(nobackground=True).get_style_defs('.highlight')  # type: ignore[no-untyped-call]
+     if CODE_HIGHLIGHT else '') + \
     """
     h2 { font-size: x-large }
 

src/nominatim_api/query_preprocessing/normalize.py

@@ -27,5 +27,5 @@ def create(config: QueryConfig) -> QueryProcessingFunc:
 
     return lambda phrases: list(
         filter(lambda p: p.text,
-               (Phrase(p.ptype, cast(str, normalizer.transliterate(p.text)))
+               (Phrase(p.ptype, cast(str, normalizer.transliterate(p.text)).strip('-: '))
                 for p in phrases)))

src/nominatim_api/query_preprocessing/regex_replace.py

@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2025 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+This preprocessor replaces values in a given input based on pre-defined regex rules.
+
+Arguments:
+    pattern: Regex pattern to be applied on the input
+    replace: The string that it is to be replaced with
+"""
+from typing import List
+import re
+
+from .config import QueryConfig
+from .base import QueryProcessingFunc
+from ..search.query import Phrase
+
+
+class _GenericPreprocessing:
+    """Perform replacements to input phrases using custom regex patterns."""
+
+    def __init__(self, config: QueryConfig) -> None:
+        """Initialise the _GenericPreprocessing class with patterns from the ICU config file."""
+        self.config = config
+
+        match_patterns = self.config.get('replacements', 'Key not found')
+        self.compiled_patterns = [
+            (re.compile(item['pattern']), item['replace']) for item in match_patterns
+            ]
+
+    def split_phrase(self, phrase: Phrase) -> Phrase:
+        """This function performs replacements on the given text using regex patterns."""
+        for item in self.compiled_patterns:
+            phrase.text = item[0].sub(item[1], phrase.text)
+
+        return phrase
+
+    def __call__(self, phrases: List[Phrase]) -> List[Phrase]:
+        """
+        Return the final Phrase list.
+        Returns an empty list if there is nothing left after split_phrase.
+        """
+        result = [p for p in map(self.split_phrase, phrases) if p.text.strip()]
+        return result
+
+
+def create(config: QueryConfig) -> QueryProcessingFunc:
+    """ Create a function for generic preprocessing."""
+    return _GenericPreprocessing(config)

src/nominatim_api/search/__init__.py

@@ -9,5 +9,12 @@ Module for forward search.
 """
 from .geocoder import (ForwardGeocoder as ForwardGeocoder)
 from .query import (Phrase as Phrase,
-                    PhraseType as PhraseType)
+                    PHRASE_ANY as PHRASE_ANY,
+                    PHRASE_AMENITY as PHRASE_AMENITY,
+                    PHRASE_STREET as PHRASE_STREET,
+                    PHRASE_CITY as PHRASE_CITY,
+                    PHRASE_COUNTY as PHRASE_COUNTY,
+                    PHRASE_STATE as PHRASE_STATE,
+                    PHRASE_POSTCODE as PHRASE_POSTCODE,
+                    PHRASE_COUNTRY as PHRASE_COUNTRY)
 from .query_analyzer_factory import (make_query_analyzer as make_query_analyzer)

src/nominatim_api/search/db_search_builder.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Conversion from token assignment to an abstract DB search.
@@ -11,7 +11,7 @@ from typing import Optional, List, Tuple, Iterator, Dict
 import heapq
 
 from ..types import SearchDetails, DataLayer
-from .query import QueryStruct, Token, TokenType, TokenRange, BreakType
+from . import query as qmod
 from .token_assignment import TokenAssignment
 from . import db_search_fields as dbf
 from . import db_searches as dbs
@@ -51,7 +51,7 @@ class SearchBuilder:
     """ Build the abstract search queries from token assignments.
     """
 
-    def __init__(self, query: QueryStruct, details: SearchDetails) -> None:
+    def __init__(self, query: qmod.QueryStruct, details: SearchDetails) -> None:
         self.query = query
         self.details = details
 
@@ -97,7 +97,7 @@ class SearchBuilder:
                 builder = self.build_poi_search(sdata)
             elif assignment.housenumber:
                 hnr_tokens = self.query.get_tokens(assignment.housenumber,
-                                                   TokenType.HOUSENUMBER)
+                                                   qmod.TOKEN_HOUSENUMBER)
                 builder = self.build_housenumber_search(sdata, hnr_tokens, assignment.address)
             else:
                 builder = self.build_special_search(sdata, assignment.address,
@@ -128,7 +128,7 @@ class SearchBuilder:
             yield dbs.PoiSearch(sdata)
 
     def build_special_search(self, sdata: dbf.SearchData,
-                             address: List[TokenRange],
+                             address: List[qmod.TokenRange],
                              is_category: bool) -> Iterator[dbs.AbstractSearch]:
         """ Build abstract search queries for searches that do not involve
             a named place.
@@ -146,13 +146,12 @@ class SearchBuilder:
             if address:
                 sdata.lookups = [dbf.FieldLookup('nameaddress_vector',
                                                  [t.token for r in address
-                                                  for t in self.query.get_partials_list(r)],
+                                                  for t in self.query.iter_partials(r)],
                                                  lookups.Restrict)]
-                penalty += 0.2
             yield dbs.PostcodeSearch(penalty, sdata)
 
-    def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[Token],
-                                 address: List[TokenRange]) -> Iterator[dbs.AbstractSearch]:
+    def build_housenumber_search(self, sdata: dbf.SearchData, hnrs: List[qmod.Token],
+                                 address: List[qmod.TokenRange]) -> Iterator[dbs.AbstractSearch]:
         """ Build a simple address search for special entries where the
             housenumber is the main name token.
         """
@@ -160,7 +159,7 @@ class SearchBuilder:
         expected_count = sum(t.count for t in hnrs)
 
         partials = {t.token: t.addr_count for trange in address
-                    for t in self.query.get_partials_list(trange)}
+                    for t in self.query.iter_partials(trange)}
 
         if not partials:
             # can happen when none of the partials is indexed
@@ -174,7 +173,7 @@ class SearchBuilder:
                                                  list(partials), lookups.LookupAll))
         else:
             addr_fulls = [t.token for t
-                          in self.query.get_tokens(address[0], TokenType.WORD)]
+                          in self.query.get_tokens(address[0], qmod.TOKEN_WORD)]
             if len(addr_fulls) > 5:
                 return
             sdata.lookups.append(
@@ -184,7 +183,7 @@ class SearchBuilder:
         yield dbs.PlaceSearch(0.05, sdata, expected_count)
 
     def build_name_search(self, sdata: dbf.SearchData,
-                          name: TokenRange, address: List[TokenRange],
+                          name: qmod.TokenRange, address: List[qmod.TokenRange],
                           is_category: bool) -> Iterator[dbs.AbstractSearch]:
         """ Build abstract search queries for simple name or address searches.
         """
@@ -197,38 +196,38 @@ class SearchBuilder:
                 sdata.lookups = lookup
                 yield dbs.PlaceSearch(penalty + name_penalty, sdata, count)
 
-    def yield_lookups(self, name: TokenRange, address: List[TokenRange]
+    def yield_lookups(self, name: qmod.TokenRange, address: List[qmod.TokenRange]
                       ) -> Iterator[Tuple[float, int, List[dbf.FieldLookup]]]:
         """ Yield all variants how the given name and address should best
             be searched for. This takes into account how frequent the terms
             are and tries to find a lookup that optimizes index use.
         """
         penalty = 0.0  # extra penalty
-        name_partials = {t.token: t for t in self.query.get_partials_list(name)}
+        name_partials = {t.token: t for t in self.query.iter_partials(name)}
 
-        addr_partials = [t for r in address for t in self.query.get_partials_list(r)]
+        addr_partials = [t for r in address for t in self.query.iter_partials(r)]
         addr_tokens = list({t.token for t in addr_partials})
 
-        exp_count = min(t.count for t in name_partials.values()) / (2**(len(name_partials) - 1))
+        exp_count = min(t.count for t in name_partials.values()) / (3**(len(name_partials) - 1))
 
         if (len(name_partials) > 3 or exp_count < 8000):
             yield penalty, exp_count, dbf.lookup_by_names(list(name_partials.keys()), addr_tokens)
             return
 
-        addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 30000
+        addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 50000
         # Partial term to frequent. Try looking up by rare full names first.
-        name_fulls = self.query.get_tokens(name, TokenType.WORD)
+        name_fulls = self.query.get_tokens(name, qmod.TOKEN_WORD)
         if name_fulls:
             fulls_count = sum(t.count for t in name_fulls)
 
-            if fulls_count < 50000 or addr_count < 30000:
+            if fulls_count < 80000 or addr_count < 50000:
                 yield penalty, fulls_count / (2**len(addr_tokens)), \
                     self.get_full_name_ranking(name_fulls, addr_partials,
                                                fulls_count > 30000 / max(1, len(addr_tokens)))
 
         # To catch remaining results, lookup by name and address
         # We only do this if there is a reasonable number of results expected.
-        exp_count = exp_count / (2**len(addr_tokens)) if addr_tokens else exp_count
+        exp_count /= 2**len(addr_tokens)
         if exp_count < 10000 and addr_count < 20000:
             penalty += 0.35 * max(1 if name_fulls else 0.1,
                                   5 - len(name_partials) - len(addr_tokens))
@@ -236,7 +235,7 @@ class SearchBuilder:
                 self.get_name_address_ranking(list(name_partials.keys()), addr_partials)
 
     def get_name_address_ranking(self, name_tokens: List[int],
-                                 addr_partials: List[Token]) -> List[dbf.FieldLookup]:
+                                 addr_partials: List[qmod.Token]) -> List[dbf.FieldLookup]:
         """ Create a ranking expression looking up by name and address.
         """
         lookup = [dbf.FieldLookup('name_vector', name_tokens, lookups.LookupAll)]
@@ -258,23 +257,16 @@ class SearchBuilder:
 
         return lookup
 
-    def get_full_name_ranking(self, name_fulls: List[Token], addr_partials: List[Token],
+    def get_full_name_ranking(self, name_fulls: List[qmod.Token], addr_partials: List[qmod.Token],
                               use_lookup: bool) -> List[dbf.FieldLookup]:
         """ Create a ranking expression with full name terms and
             additional address lookup. When 'use_lookup' is true, then
             address lookups will use the index, when the occurrences are not
             too many.
         """
-        # At this point drop unindexed partials from the address.
-        # This might yield wrong results, nothing we can do about that.
         if use_lookup:
             addr_restrict_tokens = []
-            addr_lookup_tokens = []
-            for t in addr_partials:
-                if t.addr_count > 20000:
-                    addr_restrict_tokens.append(t.token)
-                else:
-                    addr_lookup_tokens.append(t.token)
+            addr_lookup_tokens = [t.token for t in addr_partials]
         else:
             addr_restrict_tokens = [t.token for t in addr_partials]
             addr_lookup_tokens = []
@@ -282,19 +274,18 @@ class SearchBuilder:
         return dbf.lookup_by_any_name([t.token for t in name_fulls],
                                       addr_restrict_tokens, addr_lookup_tokens)
 
-    def get_name_ranking(self, trange: TokenRange,
+    def get_name_ranking(self, trange: qmod.TokenRange,
                          db_field: str = 'name_vector') -> dbf.FieldRanking:
         """ Create a ranking expression for a name term in the given range.
         """
-        name_fulls = self.query.get_tokens(trange, TokenType.WORD)
+        name_fulls = self.query.get_tokens(trange, qmod.TOKEN_WORD)
         ranks = [dbf.RankedTokens(t.penalty, [t.token]) for t in name_fulls]
         ranks.sort(key=lambda r: r.penalty)
         # Fallback, sum of penalty for partials
-        name_partials = self.query.get_partials_list(trange)
-        default = sum(t.penalty for t in name_partials) + 0.2
+        default = sum(t.penalty for t in self.query.iter_partials(trange)) + 0.2
         return dbf.FieldRanking(db_field, default, ranks)
 
-    def get_addr_ranking(self, trange: TokenRange) -> dbf.FieldRanking:
+    def get_addr_ranking(self, trange: qmod.TokenRange) -> dbf.FieldRanking:
         """ Create a list of ranking expressions for an address term
             for the given ranges.
         """
@@ -304,35 +295,35 @@ class SearchBuilder:
 
         while todo:
             neglen, pos, rank = heapq.heappop(todo)
+            # partial node
+            partial = self.query.nodes[pos].partial
+            if partial is not None:
+                if pos + 1 < trange.end:
+                    penalty = rank.penalty + partial.penalty \
+                              + PENALTY_WORDCHANGE[self.query.nodes[pos + 1].btype]
+                    heapq.heappush(todo, (neglen - 1, pos + 1,
+                                   dbf.RankedTokens(penalty, rank.tokens)))
+                else:
+                    ranks.append(dbf.RankedTokens(rank.penalty + partial.penalty,
+                                                  rank.tokens))
+            # full words
             for tlist in self.query.nodes[pos].starting:
-                if tlist.ttype in (TokenType.PARTIAL, TokenType.WORD):
+                if tlist.ttype == qmod.TOKEN_WORD:
                     if tlist.end < trange.end:
                         chgpenalty = PENALTY_WORDCHANGE[self.query.nodes[tlist.end].btype]
-                        if tlist.ttype == TokenType.PARTIAL:
-                            penalty = rank.penalty + chgpenalty \
-                                      + max(t.penalty for t in tlist.tokens)
+                        for t in tlist.tokens:
                             heapq.heappush(todo, (neglen - 1, tlist.end,
-                                                  dbf.RankedTokens(penalty, rank.tokens)))
-                        else:
-                            for t in tlist.tokens:
-                                heapq.heappush(todo, (neglen - 1, tlist.end,
-                                                      rank.with_token(t, chgpenalty)))
+                                                  rank.with_token(t, chgpenalty)))
                     elif tlist.end == trange.end:
-                        if tlist.ttype == TokenType.PARTIAL:
-                            ranks.append(dbf.RankedTokens(rank.penalty
-                                                          + max(t.penalty for t in tlist.tokens),
-                                                          rank.tokens))
-                        else:
-                            ranks.extend(rank.with_token(t, 0.0) for t in tlist.tokens)
-                        if len(ranks) >= 10:
-                            # Too many variants, bail out and only add
-                            # Worst-case Fallback: sum of penalty of partials
-                            name_partials = self.query.get_partials_list(trange)
-                            default = sum(t.penalty for t in name_partials) + 0.2
-                            ranks.append(dbf.RankedTokens(rank.penalty + default, []))
-                            # Bail out of outer loop
-                            todo.clear()
-                            break
+                        ranks.extend(rank.with_token(t, 0.0) for t in tlist.tokens)
+
+            if len(ranks) >= 10:
+                # Too many variants, bail out and only add
+                # Worst-case Fallback: sum of penalty of partials
+                default = sum(t.penalty for t in self.query.iter_partials(trange)) + 0.2
+                ranks.append(dbf.RankedTokens(rank.penalty + default, []))
+                # Bail out of outer loop
+                break
 
         ranks.sort(key=lambda r: len(r.tokens))
         default = ranks[0].penalty + 0.3
@@ -358,11 +349,11 @@ class SearchBuilder:
         if assignment.housenumber:
             sdata.set_strings('housenumbers',
                               self.query.get_tokens(assignment.housenumber,
-                                                    TokenType.HOUSENUMBER))
+                                                    qmod.TOKEN_HOUSENUMBER))
         if assignment.postcode:
             sdata.set_strings('postcodes',
                               self.query.get_tokens(assignment.postcode,
-                                                    TokenType.POSTCODE))
+                                                    qmod.TOKEN_POSTCODE))
         if assignment.qualifier:
             tokens = self.get_qualifier_tokens(assignment.qualifier)
             if not tokens:
@@ -387,23 +378,23 @@ class SearchBuilder:
 
         return sdata
 
-    def get_country_tokens(self, trange: TokenRange) -> List[Token]:
+    def get_country_tokens(self, trange: qmod.TokenRange) -> List[qmod.Token]:
         """ Return the list of country tokens for the given range,
             optionally filtered by the country list from the details
             parameters.
         """
-        tokens = self.query.get_tokens(trange, TokenType.COUNTRY)
+        tokens = self.query.get_tokens(trange, qmod.TOKEN_COUNTRY)
         if self.details.countries:
             tokens = [t for t in tokens if t.lookup_word in self.details.countries]
 
         return tokens
 
-    def get_qualifier_tokens(self, trange: TokenRange) -> List[Token]:
+    def get_qualifier_tokens(self, trange: qmod.TokenRange) -> List[qmod.Token]:
         """ Return the list of qualifier tokens for the given range,
             optionally filtered by the qualifier list from the details
             parameters.
         """
-        tokens = self.query.get_tokens(trange, TokenType.QUALIFIER)
+        tokens = self.query.get_tokens(trange, qmod.TOKEN_QUALIFIER)
         if self.details.categories:
             tokens = [t for t in tokens if t.get_category() in self.details.categories]
 
@@ -416,7 +407,7 @@ class SearchBuilder:
         """
         if assignment.near_item:
             tokens: Dict[Tuple[str, str], float] = {}
-            for t in self.query.get_tokens(assignment.near_item, TokenType.NEAR_ITEM):
+            for t in self.query.get_tokens(assignment.near_item, qmod.TOKEN_NEAR_ITEM):
                 cat = t.get_category()
                 # The category of a near search will be that of near_item.
                 # Thus, if search is restricted to a category parameter,
@@ -430,11 +421,11 @@ class SearchBuilder:
 
 
 PENALTY_WORDCHANGE = {
-    BreakType.START: 0.0,
-    BreakType.END: 0.0,
-    BreakType.PHRASE: 0.0,
-    BreakType.SOFT_PHRASE: 0.0,
-    BreakType.WORD: 0.1,
-    BreakType.PART: 0.2,
-    BreakType.TOKEN: 0.4
+    qmod.BREAK_START: 0.0,
+    qmod.BREAK_END: 0.0,
+    qmod.BREAK_PHRASE: 0.0,
+    qmod.BREAK_SOFT_PHRASE: 0.0,
+    qmod.BREAK_WORD: 0.1,
+    qmod.BREAK_PART: 0.2,
+    qmod.BREAK_TOKEN: 0.4
 }

src/nominatim_api/search/db_searches.py

@@ -581,9 +581,13 @@ class PostcodeSearch(AbstractSearch):
                      .where((tsearch.c.name_vector + tsearch.c.nameaddress_vector)
                             .contains(sa.type_coerce(self.lookups[0].tokens,
                                                      IntArray)))
+            # Do NOT add rerank penalties based on the address terms.
+            # The standard rerank penalty only checks the address vector
+            # while terms may appear in name and address vector. This would
+            # lead to overly high penalties.
+            # We assume that a postcode is precise enough to not require
+            # additional full name matches.
 
-        for ranking in self.rankings:
-            penalty += ranking.sql_penalty(conn.t.search_name)
         penalty += sa.case(*((t.c.postcode == v, p) for v, p in self.postcodes),
                            else_=1.0)
 

src/nominatim_api/search/geocoder.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Public interface to the search code.
@@ -50,6 +50,9 @@ class ForwardGeocoder:
             self.query_analyzer = await make_query_analyzer(self.conn)
 
         query = await self.query_analyzer.analyze_query(phrases)
+        query.compute_direction_penalty()
+        log().var_dump('Query direction penalty',
+                       lambda: f"[{'LR' if query.dir_penalty < 0 else 'RL'}] {query.dir_penalty}")
 
         searches: List[AbstractSearch] = []
         if query.num_token_slots() > 0:
@@ -115,17 +118,20 @@ class ForwardGeocoder:
         """ Remove badly matching results, sort by ranking and
             limit to the configured number of results.
         """
-        if results:
-            results.sort(key=lambda r: (r.ranking, 0 if r.bbox is None else -r.bbox.area))
-            min_rank = results[0].rank_search
-            min_ranking = results[0].ranking
-            results = SearchResults(r for r in results
-                                    if (r.ranking + 0.03 * (r.rank_search - min_rank)
-                                        < min_ranking + 0.5))
+        results.sort(key=lambda r: (r.ranking, 0 if r.bbox is None else -r.bbox.area))
 
-            results = SearchResults(results[:self.limit])
+        final = SearchResults()
+        min_rank = results[0].rank_search
+        min_ranking = results[0].ranking
 
-        return results
+        for r in results:
+            if r.ranking + 0.03 * (r.rank_search - min_rank) < min_ranking + 0.5:
+                final.append(r)
+                min_rank = min(r.rank_search, min_rank)
+            if len(final) == self.limit:
+                break
+
+        return final
 
     def rerank_by_query(self, query: QueryStruct, results: SearchResults) -> None:
         """ Adjust the accuracy of the localized result according to how well
@@ -150,17 +156,16 @@ class ForwardGeocoder:
             if not words:
                 continue
             for qword in qwords:
-                wdist = max(difflib.SequenceMatcher(a=qword, b=w).quick_ratio() for w in words)
-                if wdist < 0.5:
-                    distance += len(qword)
-                else:
-                    distance += (1.0 - wdist) * len(qword)
+                # only add distance penalty if there is no perfect match
+                if qword not in words:
+                    wdist = max(difflib.SequenceMatcher(a=qword, b=w).quick_ratio() for w in words)
+                    distance += len(qword) if wdist < 0.4 else 1
             # Compensate for the fact that country names do not get a
             # match penalty yet by the tokenizer.
             # Temporary hack that needs to be removed!
             if result.rank_address == 4:
                 distance *= 2
-            result.accuracy += distance * 0.4 / sum(len(w) for w in qwords)
+            result.accuracy += distance * 0.3 / sum(len(w) for w in qwords)
 
     async def lookup_pois(self, categories: List[Tuple[str, str]],
                           phrases: List[Phrase]) -> SearchResults:
@@ -208,9 +213,10 @@ class ForwardGeocoder:
             results = self.pre_filter_results(results)
             await add_result_details(self.conn, results, self.params)
             log().result_dump('Preliminary Results', ((r.accuracy, r) for r in results))
-            self.rerank_by_query(query, results)
-            log().result_dump('Results after reranking', ((r.accuracy, r) for r in results))
-            results = self.sort_and_cut_results(results)
+            if len(results) > 1:
+                self.rerank_by_query(query, results)
+                log().result_dump('Results after reranking', ((r.accuracy, r) for r in results))
+                results = self.sort_and_cut_results(results)
             log().result_dump('Final Results', ((r.accuracy, r) for r in results))
 
         return results
@@ -238,7 +244,7 @@ def _dump_searches(searches: List[AbstractSearch], query: QueryStruct,
         if not lk:
             return ''
 
-        return f"{lk.lookup_type}({lk.column}{tk(lk.tokens)})"
+        return f"{lk.lookup_type.__name__}({lk.column}{tk(lk.tokens)})"
 
     def fmt_cstr(c: Any) -> str:
         if not c:

src/nominatim_api/search/icu_tokenizer.py

@@ -2,13 +2,12 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Implementation of query analysis for the ICU tokenizer.
 """
 from typing import Tuple, Dict, List, Optional, Iterator, Any, cast
-from collections import defaultdict
 import dataclasses
 import difflib
 import re
@@ -25,62 +24,30 @@ from ..connection import SearchConnection
 from ..logging import log
 from . import query as qmod
 from ..query_preprocessing.config import QueryConfig
+from ..query_preprocessing.base import QueryProcessingFunc
 from .query_analyzer_factory import AbstractQueryAnalyzer
+from .postcode_parser import PostcodeParser
 
 
 DB_TO_TOKEN_TYPE = {
-    'W': qmod.TokenType.WORD,
-    'w': qmod.TokenType.PARTIAL,
-    'H': qmod.TokenType.HOUSENUMBER,
-    'P': qmod.TokenType.POSTCODE,
-    'C': qmod.TokenType.COUNTRY
+    'W': qmod.TOKEN_WORD,
+    'w': qmod.TOKEN_PARTIAL,
+    'H': qmod.TOKEN_HOUSENUMBER,
+    'P': qmod.TOKEN_POSTCODE,
+    'C': qmod.TOKEN_COUNTRY
 }
 
 PENALTY_IN_TOKEN_BREAK = {
-     qmod.BreakType.START: 0.5,
-     qmod.BreakType.END: 0.5,
-     qmod.BreakType.PHRASE: 0.5,
-     qmod.BreakType.SOFT_PHRASE: 0.5,
-     qmod.BreakType.WORD: 0.1,
-     qmod.BreakType.PART: 0.0,
-     qmod.BreakType.TOKEN: 0.0
+     qmod.BREAK_START: 0.5,
+     qmod.BREAK_END: 0.5,
+     qmod.BREAK_PHRASE: 0.5,
+     qmod.BREAK_SOFT_PHRASE: 0.5,
+     qmod.BREAK_WORD: 0.1,
+     qmod.BREAK_PART: 0.0,
+     qmod.BREAK_TOKEN: 0.0
 }
 
 
-@dataclasses.dataclass
-class QueryPart:
-    """ Normalized and transliterated form of a single term in the query.
-        When the term came out of a split during the transliteration,
-        the normalized string is the full word before transliteration.
-        The word number keeps track of the word before transliteration
-        and can be used to identify partial transliterated terms.
-        Penalty is the break penalty for the break following the token.
-    """
-    token: str
-    normalized: str
-    word_number: int
-    penalty: float
-
-
-QueryParts = List[QueryPart]
-WordDict = Dict[str, List[qmod.TokenRange]]
-
-
-def yield_words(terms: List[QueryPart], start: int) -> Iterator[Tuple[str, qmod.TokenRange]]:
-    """ Return all combinations of words in the terms list after the
-        given position.
-    """
-    total = len(terms)
-    for first in range(start, total):
-        word = terms[first].token
-        penalty = PENALTY_IN_TOKEN_BREAK[qmod.BreakType.WORD]
-        yield word, qmod.TokenRange(first, first + 1, penalty=penalty)
-        for last in range(first + 1, min(first + 20, total)):
-            word = ' '.join((word, terms[last].token))
-            penalty += terms[last - 1].penalty
-            yield word, qmod.TokenRange(first, last + 1, penalty=penalty)
-
-
 @dataclasses.dataclass
 class ICUToken(qmod.Token):
     """ Specialised token for ICU tokenizer.
@@ -146,60 +113,51 @@ class ICUToken(qmod.Token):
                         addr_count=max(1, addr_count))
 
 
-class ICUQueryAnalyzer(AbstractQueryAnalyzer):
-    """ Converter for query strings into a tokenized query
-        using the tokens created by a ICU tokenizer.
-    """
-    def __init__(self, conn: SearchConnection) -> None:
-        self.conn = conn
+@dataclasses.dataclass
+class ICUAnalyzerConfig:
+    postcode_parser: PostcodeParser
+    normalizer: Transliterator
+    transliterator: Transliterator
+    preprocessors: List[QueryProcessingFunc]
 
-    async def setup(self) -> None:
-        """ Set up static data structures needed for the analysis.
-        """
-        async def _make_normalizer() -> Any:
-            rules = await self.conn.get_property('tokenizer_import_normalisation')
-            return Transliterator.createFromRules("normalization", rules)
+    @staticmethod
+    async def create(conn: SearchConnection) -> 'ICUAnalyzerConfig':
+        rules = await conn.get_property('tokenizer_import_normalisation')
+        normalizer = Transliterator.createFromRules("normalization", rules)
 
-        self.normalizer = await self.conn.get_cached_value('ICUTOK', 'normalizer',
-                                                           _make_normalizer)
+        rules = await conn.get_property('tokenizer_import_transliteration')
+        transliterator = Transliterator.createFromRules("transliteration", rules)
 
-        async def _make_transliterator() -> Any:
-            rules = await self.conn.get_property('tokenizer_import_transliteration')
-            return Transliterator.createFromRules("transliteration", rules)
-
-        self.transliterator = await self.conn.get_cached_value('ICUTOK', 'transliterator',
-                                                               _make_transliterator)
-
-        await self._setup_preprocessing()
-
-        if 'word' not in self.conn.t.meta.tables:
-            sa.Table('word', self.conn.t.meta,
-                     sa.Column('word_id', sa.Integer),
-                     sa.Column('word_token', sa.Text, nullable=False),
-                     sa.Column('type', sa.Text, nullable=False),
-                     sa.Column('word', sa.Text),
-                     sa.Column('info', Json))
-
-    async def _setup_preprocessing(self) -> None:
-        """ Load the rules for preprocessing and set up the handlers.
-        """
-
-        rules = self.conn.config.load_sub_configuration('icu_tokenizer.yaml',
-                                                        config='TOKENIZER_CONFIG')
-        preprocessing_rules = rules.get('query-preprocessing', [])
-
-        self.preprocessors = []
+        preprocessing_rules = conn.config.load_sub_configuration('icu_tokenizer.yaml',
+                                                                 config='TOKENIZER_CONFIG')\
+                                         .get('query-preprocessing', [])
 
+        preprocessors: List[QueryProcessingFunc] = []
         for func in preprocessing_rules:
             if 'step' not in func:
                 raise UsageError("Preprocessing rule is missing the 'step' attribute.")
             if not isinstance(func['step'], str):
                 raise UsageError("'step' attribute must be a simple string.")
 
-            module = self.conn.config.load_plugin_module(
+            module = conn.config.load_plugin_module(
                         func['step'], 'nominatim_api.query_preprocessing')
-            self.preprocessors.append(
-                module.create(QueryConfig(func).set_normalizer(self.normalizer)))
+            preprocessors.append(
+                module.create(QueryConfig(func).set_normalizer(normalizer)))
+
+        return ICUAnalyzerConfig(PostcodeParser(conn.config),
+                                 normalizer, transliterator, preprocessors)
+
+
+class ICUQueryAnalyzer(AbstractQueryAnalyzer):
+    """ Converter for query strings into a tokenized query
+        using the tokens created by a ICU tokenizer.
+    """
+    def __init__(self, conn: SearchConnection, config: ICUAnalyzerConfig) -> None:
+        self.conn = conn
+        self.postcode_parser = config.postcode_parser
+        self.normalizer = config.normalizer
+        self.transliterator = config.transliterator
+        self.preprocessors = config.preprocessors
 
     async def analyze_query(self, phrases: List[qmod.Phrase]) -> qmod.QueryStruct:
         """ Analyze the given list of phrases and return the
@@ -214,8 +172,9 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
         if not query.source:
             return query
 
-        parts, words = self.split_query(query)
-        log().var_dump('Transliterated query', lambda: _dump_transliterated(query, parts))
+        self.split_query(query)
+        log().var_dump('Transliterated query', lambda: query.get_transliterated_query())
+        words = query.extract_words(base_penalty=PENALTY_IN_TOKEN_BREAK[qmod.BREAK_WORD])
 
         for row in await self.lookup_in_db(list(words.keys())):
             for trange in words[row.word_token]:
@@ -223,17 +182,24 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
                 if row.type == 'S':
                     if row.info['op'] in ('in', 'near'):
                         if trange.start == 0:
-                            query.add_token(trange, qmod.TokenType.NEAR_ITEM, token)
+                            query.add_token(trange, qmod.TOKEN_NEAR_ITEM, token)
                     else:
                         if trange.start == 0 and trange.end == query.num_token_slots():
-                            query.add_token(trange, qmod.TokenType.NEAR_ITEM, token)
+                            query.add_token(trange, qmod.TOKEN_NEAR_ITEM, token)
                         else:
-                            query.add_token(trange, qmod.TokenType.QUALIFIER, token)
+                            query.add_token(trange, qmod.TOKEN_QUALIFIER, token)
                 else:
                     query.add_token(trange, DB_TO_TOKEN_TYPE[row.type], token)
 
-        self.add_extra_tokens(query, parts)
-        self.rerank_tokens(query, parts)
+        self.add_extra_tokens(query)
+        for start, end, pc in self.postcode_parser.parse(query):
+            term = ' '.join(n.term_lookup for n in query.nodes[start + 1:end + 1])
+            query.add_token(qmod.TokenRange(start, end),
+                            qmod.TOKEN_POSTCODE,
+                            ICUToken(penalty=0.1, token=0, count=1, addr_count=1,
+                                     lookup_word=pc, word_token=term,
+                                     info=None))
+        self.rerank_tokens(query)
 
         log().table_dump('Word tokens', _dump_word_tokens(query))
 
@@ -244,19 +210,11 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
             standardized form search will work with. All information removed
             at this stage is inevitably lost.
         """
-        return cast(str, self.normalizer.transliterate(text))
+        return cast(str, self.normalizer.transliterate(text)).strip('-: ')
 
-    def split_query(self, query: qmod.QueryStruct) -> Tuple[QueryParts, WordDict]:
+    def split_query(self, query: qmod.QueryStruct) -> None:
         """ Transliterate the phrases and split them into tokens.
-
-            Returns the list of transliterated tokens together with their
-            normalized form and a dictionary of words for lookup together
-            with their position.
         """
-        parts: QueryParts = []
-        phrase_start = 0
-        words = defaultdict(list)
-        wordnr = 0
         for phrase in query.source:
             query.nodes[-1].ptype = phrase.ptype
             phrase_split = re.split('([ :-])', phrase.text)
@@ -271,78 +229,89 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
                 if trans:
                     for term in trans.split(' '):
                         if term:
-                            parts.append(QueryPart(term, word, wordnr,
-                                                   PENALTY_IN_TOKEN_BREAK[qmod.BreakType.TOKEN]))
-                            query.add_node(qmod.BreakType.TOKEN, phrase.ptype)
-                    query.nodes[-1].btype = qmod.BreakType(breakchar)
-                    parts[-1].penalty = PENALTY_IN_TOKEN_BREAK[qmod.BreakType(breakchar)]
-                wordnr += 1
+                            query.add_node(qmod.BREAK_TOKEN, phrase.ptype,
+                                           PENALTY_IN_TOKEN_BREAK[qmod.BREAK_TOKEN],
+                                           term, word)
+                    query.nodes[-1].adjust_break(breakchar,
+                                                 PENALTY_IN_TOKEN_BREAK[breakchar])
 
-            for word, wrange in yield_words(parts, phrase_start):
-                words[word].append(wrange)
-
-            phrase_start = len(parts)
-        query.nodes[-1].btype = qmod.BreakType.END
-
-        return parts, words
+        query.nodes[-1].adjust_break(qmod.BREAK_END, PENALTY_IN_TOKEN_BREAK[qmod.BREAK_END])
 
     async def lookup_in_db(self, words: List[str]) -> 'sa.Result[Any]':
         """ Return the token information from the database for the
             given word tokens.
+
+            This function excludes postcode tokens
         """
         t = self.conn.t.meta.tables['word']
-        return await self.conn.execute(t.select().where(t.c.word_token.in_(words)))
+        return await self.conn.execute(t.select()
+                                        .where(t.c.word_token.in_(words))
+                                        .where(t.c.type != 'P'))
 
-    def add_extra_tokens(self, query: qmod.QueryStruct, parts: QueryParts) -> None:
+    def add_extra_tokens(self, query: qmod.QueryStruct) -> None:
         """ Add tokens to query that are not saved in the database.
         """
-        for part, node, i in zip(parts, query.nodes, range(1000)):
-            if len(part.token) <= 4 and part.token.isdigit()\
-               and not node.has_tokens(i+1, qmod.TokenType.HOUSENUMBER):
-                query.add_token(qmod.TokenRange(i, i+1), qmod.TokenType.HOUSENUMBER,
+        need_hnr = False
+        for i, node in enumerate(query.nodes):
+            is_full_token = node.btype not in (qmod.BREAK_TOKEN, qmod.BREAK_PART)
+            if need_hnr and is_full_token \
+                    and len(node.term_normalized) <= 4 and node.term_normalized.isdigit():
+                query.add_token(qmod.TokenRange(i-1, i), qmod.TOKEN_HOUSENUMBER,
                                 ICUToken(penalty=0.5, token=0,
-                                         count=1, addr_count=1, lookup_word=part.token,
-                                         word_token=part.token, info=None))
+                                         count=1, addr_count=1,
+                                         lookup_word=node.term_lookup,
+                                         word_token=node.term_lookup, info=None))
 
-    def rerank_tokens(self, query: qmod.QueryStruct, parts: QueryParts) -> None:
+            need_hnr = is_full_token and not node.has_tokens(i+1, qmod.TOKEN_HOUSENUMBER)
+
+    def rerank_tokens(self, query: qmod.QueryStruct) -> None:
         """ Add penalties to tokens that depend on presence of other token.
         """
-        for i, node, tlist in query.iter_token_lists():
-            if tlist.ttype == qmod.TokenType.POSTCODE:
-                for repl in node.starting:
-                    if repl.end == tlist.end and repl.ttype != qmod.TokenType.POSTCODE \
-                       and (repl.ttype != qmod.TokenType.HOUSENUMBER
-                            or len(tlist.tokens[0].lookup_word) > 4):
-                        repl.add_penalty(0.39)
-            elif (tlist.ttype == qmod.TokenType.HOUSENUMBER
-                  and len(tlist.tokens[0].lookup_word) <= 3):
-                if any(c.isdigit() for c in tlist.tokens[0].lookup_word):
-                    for repl in node.starting:
-                        if repl.end == tlist.end and repl.ttype != qmod.TokenType.HOUSENUMBER:
-                            repl.add_penalty(0.5 - tlist.tokens[0].penalty)
-            elif tlist.ttype not in (qmod.TokenType.COUNTRY, qmod.TokenType.PARTIAL):
-                norm = parts[i].normalized
-                for j in range(i + 1, tlist.end):
-                    if parts[j - 1].word_number != parts[j].word_number:
-                        norm += '  ' + parts[j].normalized
-                for token in tlist.tokens:
-                    cast(ICUToken, token).rematch(norm)
+        for start, end, tlist in query.iter_tokens_by_edge():
+            if len(tlist) > 1:
+                # If it looks like a Postcode, give preference.
+                if qmod.TOKEN_POSTCODE in tlist:
+                    for ttype, tokens in tlist.items():
+                        if ttype != qmod.TOKEN_POSTCODE and \
+                               (ttype != qmod.TOKEN_HOUSENUMBER or
+                                start + 1 > end or
+                                len(query.nodes[end].term_lookup) > 4):
+                            for token in tokens:
+                                token.penalty += 0.39
 
+                # If it looks like a simple housenumber, prefer that.
+                if qmod.TOKEN_HOUSENUMBER in tlist:
+                    hnr_lookup = tlist[qmod.TOKEN_HOUSENUMBER][0].lookup_word
+                    if len(hnr_lookup) <= 3 and any(c.isdigit() for c in hnr_lookup):
+                        penalty = 0.5 - tlist[qmod.TOKEN_HOUSENUMBER][0].penalty
+                        for ttype, tokens in tlist.items():
+                            if ttype != qmod.TOKEN_HOUSENUMBER:
+                                for token in tokens:
+                                    token.penalty += penalty
 
-def _dump_transliterated(query: qmod.QueryStruct, parts: QueryParts) -> str:
-    out = query.nodes[0].btype.value
-    for node, part in zip(query.nodes[1:], parts):
-        out += part.token + node.btype.value
-    return out
+            # rerank tokens against the normalized form
+            norm = ' '.join(n.term_normalized for n in query.nodes[start + 1:end + 1]
+                            if n.btype != qmod.BREAK_TOKEN)
+            if not norm:
+                # Can happen when the token only covers a partial term
+                norm = query.nodes[start + 1].term_normalized
+            for ttype, tokens in tlist.items():
+                if ttype != qmod.TOKEN_COUNTRY:
+                    for token in tokens:
+                        cast(ICUToken, token).rematch(norm)
 
 
 def _dump_word_tokens(query: qmod.QueryStruct) -> Iterator[List[Any]]:
-    yield ['type', 'token', 'word_token', 'lookup_word', 'penalty', 'count', 'info']
-    for node in query.nodes:
+    yield ['type', 'from', 'to', 'token', 'word_token', 'lookup_word', 'penalty', 'count', 'info']
+    for i, node in enumerate(query.nodes):
+        if node.partial is not None:
+            t = cast(ICUToken, node.partial)
+            yield [qmod.TOKEN_PARTIAL, str(i), str(i + 1), t.token,
+                   t.word_token, t.lookup_word, t.penalty, t.count, t.info]
         for tlist in node.starting:
             for token in tlist.tokens:
                 t = cast(ICUToken, token)
-                yield [tlist.ttype.name, t.token, t.word_token or '',
+                yield [tlist.ttype, str(i), str(tlist.end), t.token, t.word_token or '',
                        t.lookup_word or '', t.penalty, t.count, t.info]
 
 
@@ -350,7 +319,17 @@ async def create_query_analyzer(conn: SearchConnection) -> AbstractQueryAnalyzer
     """ Create and set up a new query analyzer for a database based
         on the ICU tokenizer.
     """
-    out = ICUQueryAnalyzer(conn)
-    await out.setup()
+    async def _get_config() -> ICUAnalyzerConfig:
+        if 'word' not in conn.t.meta.tables:
+            sa.Table('word', conn.t.meta,
+                     sa.Column('word_id', sa.Integer),
+                     sa.Column('word_token', sa.Text, nullable=False),
+                     sa.Column('type', sa.Text, nullable=False),
+                     sa.Column('word', sa.Text),
+                     sa.Column('info', Json))
 
-    return out
+        return await ICUAnalyzerConfig.create(conn)
+
+    config = await conn.get_cached_value('ICUTOK', 'config', _get_config)
+
+    return ICUQueryAnalyzer(conn, config)

src/nominatim_api/search/postcode_parser.py

@@ -0,0 +1,104 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2025 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Handling of arbitrary postcode tokens in tokenized query string.
+"""
+from typing import Tuple, Set, Dict, List
+import re
+from collections import defaultdict
+
+import yaml
+
+from ..config import Configuration
+from . import query as qmod
+
+
+class PostcodeParser:
+    """ Pattern-based parser for postcodes in tokenized queries.
+
+        The postcode patterns are read from the country configuration.
+        The parser does currently not return country restrictions.
+    """
+
+    def __init__(self, config: Configuration) -> None:
+        # skip over includes here to avoid loading the complete country name data
+        yaml.add_constructor('!include', lambda loader, node: [],
+                             Loader=yaml.SafeLoader)
+        cdata = yaml.safe_load(config.find_config_file('country_settings.yaml')
+                                     .read_text(encoding='utf-8'))
+
+        unique_patterns: Dict[str, Dict[str, List[str]]] = {}
+        for cc, data in cdata.items():
+            if data.get('postcode'):
+                pat = data['postcode']['pattern'].replace('d', '[0-9]').replace('l', '[A-Z]')
+                out = data['postcode'].get('output')
+                if pat not in unique_patterns:
+                    unique_patterns[pat] = defaultdict(list)
+                unique_patterns[pat][out].append(cc.upper())
+
+        self.global_pattern = re.compile(
+                '(?:(?P<cc>[A-Z][A-Z])(?P<space>[ -]?))?(?P<pc>(?:(?:'
+                + ')|(?:'.join(unique_patterns) + '))[:, >].*)')
+
+        self.local_patterns = [(re.compile(f"{pat}[:, >]"), list(info.items()))
+                               for pat, info in unique_patterns.items()]
+
+    def parse(self, query: qmod.QueryStruct) -> Set[Tuple[int, int, str]]:
+        """ Parse postcodes in the given list of query tokens taking into
+            account the list of breaks from the nodes.
+
+            The result is a sequence of tuples with
+            [start node id, end node id, postcode token]
+        """
+        nodes = query.nodes
+        outcodes: Set[Tuple[int, int, str]] = set()
+
+        terms = [n.term_normalized.upper() + n.btype for n in nodes]
+        for i in range(query.num_token_slots()):
+            if nodes[i].btype in '<,: ' and nodes[i + 1].btype != '`' \
+                    and (i == 0 or nodes[i - 1].ptype != qmod.PHRASE_POSTCODE):
+                if nodes[i].ptype == qmod.PHRASE_ANY:
+                    word = terms[i + 1]
+                    if word[-1] in ' -' and nodes[i + 2].btype != '`' \
+                            and nodes[i + 1].ptype == qmod.PHRASE_ANY:
+                        word += terms[i + 2]
+                        if word[-1] in ' -' and nodes[i + 3].btype != '`' \
+                                and nodes[i + 2].ptype == qmod.PHRASE_ANY:
+                            word += terms[i + 3]
+
+                    self._match_word(word, i, False, outcodes)
+                elif nodes[i].ptype == qmod.PHRASE_POSTCODE:
+                    word = terms[i + 1]
+                    for j in range(i + 1, query.num_token_slots()):
+                        if nodes[j].ptype != qmod.PHRASE_POSTCODE:
+                            break
+                        word += terms[j + 1]
+
+                    self._match_word(word, i, True, outcodes)
+
+        return outcodes
+
+    def _match_word(self, word: str, pos: int, fullmatch: bool,
+                    outcodes: Set[Tuple[int, int, str]]) -> None:
+        # Use global pattern to check for presence of any postcode.
+        m = self.global_pattern.fullmatch(word)
+        if m:
+            # If there was a match, check against each pattern separately
+            # because multiple patterns might be machting at the end.
+            cc = m.group('cc')
+            pc_word = m.group('pc')
+            cc_spaces = len(m.group('space') or '')
+            for pattern, info in self.local_patterns:
+                lm = pattern.fullmatch(pc_word) if fullmatch else pattern.match(pc_word)
+                if lm:
+                    trange = (pos, pos + cc_spaces + sum(c in ' ,-:>' for c in lm.group(0)))
+                    for out, out_ccs in info:
+                        if cc is None or cc in out_ccs:
+                            if out:
+                                outcodes.add((*trange, lm.expand(out)))
+                            else:
+                                outcodes.add((*trange, lm.group(0)[:-1]))

src/nominatim_api/search/query.py

@@ -2,99 +2,111 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Datastructures for a tokenized query.
 """
-from typing import List, Tuple, Optional, Iterator
+from typing import Dict, List, Tuple, Optional, Iterator
 from abc import ABC, abstractmethod
+from collections import defaultdict
 import dataclasses
-import enum
+
+# Precomputed denominator for the computation of the linear regression slope
+# used to determine the query direction.
+# The x value for the regression computation will be the position of the
+# token in the query. Thus we know the x values will be [0, query length).
+# As the denominator only depends on the x values, we can pre-compute here
+# the denominatior to use for a given query length.
+# Note that query length of two or less is special cased and will not use
+# the values from this array. Thus it is not a problem that they are 0.
+LINFAC = [i * (sum(si * si for si in range(i)) - (i - 1) * i * (i - 1) / 4)
+          for i in range(50)]
 
 
-class BreakType(enum.Enum):
-    """ Type of break between tokens.
-    """
-    START = '<'
-    """ Begin of the query. """
-    END = '>'
-    """ End of the query. """
-    PHRASE = ','
-    """ Hard break between two phrases. Address parts cannot cross hard
-        phrase boundaries."""
-    SOFT_PHRASE = ':'
-    """ Likely break between two phrases. Address parts should not cross soft
-        phrase boundaries. Soft breaks can be inserted by a preprocessor
-        that is analysing the input string.
-    """
-    WORD = ' '
-    """ Break between words. """
-    PART = '-'
-    """ Break inside a word, for example a hyphen or apostrophe. """
-    TOKEN = '`'
-    """ Break created as a result of tokenization.
-        This may happen in languages without spaces between words.
+BreakType = str
+""" Type of break between tokens.
+"""
+BREAK_START = '<'
+""" Begin of the query. """
+BREAK_END = '>'
+""" End of the query. """
+BREAK_PHRASE = ','
+""" Hard break between two phrases. Address parts cannot cross hard
+    phrase boundaries."""
+BREAK_SOFT_PHRASE = ':'
+""" Likely break between two phrases. Address parts should not cross soft
+    phrase boundaries. Soft breaks can be inserted by a preprocessor
+    that is analysing the input string.
+"""
+BREAK_WORD = ' '
+""" Break between words. """
+BREAK_PART = '-'
+""" Break inside a word, for example a hyphen or apostrophe. """
+BREAK_TOKEN = '`'
+""" Break created as a result of tokenization.
+    This may happen in languages without spaces between words.
+"""
+
+
+TokenType = str
+""" Type of token.
+"""
+TOKEN_WORD = 'W'
+""" Full name of a place. """
+TOKEN_PARTIAL = 'w'
+""" Word term without breaks, does not necessarily represent a full name. """
+TOKEN_HOUSENUMBER = 'H'
+""" Housenumber term. """
+TOKEN_POSTCODE = 'P'
+""" Postal code term. """
+TOKEN_COUNTRY = 'C'
+""" Country name or reference. """
+TOKEN_QUALIFIER = 'Q'
+""" Special term used together with name (e.g. _Hotel_ Bellevue). """
+TOKEN_NEAR_ITEM = 'N'
+""" Special term used as searchable object(e.g. supermarket in ...). """
+
+
+PhraseType = int
+""" Designation of a phrase.
+"""
+PHRASE_ANY = 0
+""" No specific designation (i.e. source is free-form query). """
+PHRASE_AMENITY = 1
+""" Contains name or type of a POI. """
+PHRASE_STREET = 2
+""" Contains a street name optionally with a housenumber. """
+PHRASE_CITY = 3
+""" Contains the postal city. """
+PHRASE_COUNTY = 4
+""" Contains the equivalent of a county. """
+PHRASE_STATE = 5
+""" Contains a state or province. """
+PHRASE_POSTCODE = 6
+""" Contains a postal code. """
+PHRASE_COUNTRY = 7
+""" Contains the country name or code. """
+
+
+def _phrase_compatible_with(ptype: PhraseType, ttype: TokenType,
+                            is_full_phrase: bool) -> bool:
+    """ Check if the given token type can be used with the phrase type.
     """
+    if ptype == PHRASE_ANY:
+        return not is_full_phrase or ttype != TOKEN_QUALIFIER
+    if ptype == PHRASE_AMENITY:
+        return ttype in (TOKEN_WORD, TOKEN_PARTIAL)\
+               or (is_full_phrase and ttype == TOKEN_NEAR_ITEM)\
+               or (not is_full_phrase and ttype == TOKEN_QUALIFIER)
+    if ptype == PHRASE_STREET:
+        return ttype in (TOKEN_WORD, TOKEN_PARTIAL, TOKEN_HOUSENUMBER)
+    if ptype == PHRASE_POSTCODE:
+        return ttype == TOKEN_POSTCODE
+    if ptype == PHRASE_COUNTRY:
+        return ttype == TOKEN_COUNTRY
 
-
-class TokenType(enum.Enum):
-    """ Type of token.
-    """
-    WORD = enum.auto()
-    """ Full name of a place. """
-    PARTIAL = enum.auto()
-    """ Word term without breaks, does not necessarily represent a full name. """
-    HOUSENUMBER = enum.auto()
-    """ Housenumber term. """
-    POSTCODE = enum.auto()
-    """ Postal code term. """
-    COUNTRY = enum.auto()
-    """ Country name or reference. """
-    QUALIFIER = enum.auto()
-    """ Special term used together with name (e.g. _Hotel_ Bellevue). """
-    NEAR_ITEM = enum.auto()
-    """ Special term used as searchable object(e.g. supermarket in ...). """
-
-
-class PhraseType(enum.Enum):
-    """ Designation of a phrase.
-    """
-    NONE = 0
-    """ No specific designation (i.e. source is free-form query). """
-    AMENITY = enum.auto()
-    """ Contains name or type of a POI. """
-    STREET = enum.auto()
-    """ Contains a street name optionally with a housenumber. """
-    CITY = enum.auto()
-    """ Contains the postal city. """
-    COUNTY = enum.auto()
-    """ Contains the equivalent of a county. """
-    STATE = enum.auto()
-    """ Contains a state or province. """
-    POSTCODE = enum.auto()
-    """ Contains a postal code. """
-    COUNTRY = enum.auto()
-    """ Contains the country name or code. """
-
-    def compatible_with(self, ttype: TokenType,
-                        is_full_phrase: bool) -> bool:
-        """ Check if the given token type can be used with the phrase type.
-        """
-        if self == PhraseType.NONE:
-            return not is_full_phrase or ttype != TokenType.QUALIFIER
-        if self == PhraseType.AMENITY:
-            return ttype in (TokenType.WORD, TokenType.PARTIAL)\
-                   or (is_full_phrase and ttype == TokenType.NEAR_ITEM)\
-                   or (not is_full_phrase and ttype == TokenType.QUALIFIER)
-        if self == PhraseType.STREET:
-            return ttype in (TokenType.WORD, TokenType.PARTIAL, TokenType.HOUSENUMBER)
-        if self == PhraseType.POSTCODE:
-            return ttype == TokenType.POSTCODE
-        if self == PhraseType.COUNTRY:
-            return ttype == TokenType.COUNTRY
-
-        return ttype in (TokenType.WORD, TokenType.PARTIAL)
+    return ttype in (TOKEN_WORD, TOKEN_PARTIAL)
 
 
 @dataclasses.dataclass
@@ -171,10 +183,49 @@ class TokenList:
 @dataclasses.dataclass
 class QueryNode:
     """ A node of the query representing a break between terms.
+
+        The node also contains information on the source term
+        ending at the node. The tokens are created from this information.
     """
     btype: BreakType
     ptype: PhraseType
+
+    penalty: float
+    """ Penalty for the break at this node.
+    """
+    term_lookup: str
+    """ Transliterated term ending at this node.
+    """
+    term_normalized: str
+    """ Normalised form of term ending at this node.
+        When the token resulted from a split during transliteration,
+        then this string contains the complete source term.
+    """
+
     starting: List[TokenList] = dataclasses.field(default_factory=list)
+    """ List of all full tokens starting at this node.
+    """
+    partial: Optional[Token] = None
+    """ Base token going to the next node.
+        May be None when the query has parts for which no words are known.
+        Note that the query may still be parsable when there are other
+        types of tokens spanning over the gap.
+    """
+
+    def name_address_ratio(self) -> float:
+        """ Return the propability that the partial token belonging to
+            this node forms part of a name (as opposed of part of the address).
+        """
+        if self.partial is None:
+            return 0.5
+
+        return self.partial.count / (self.partial.count + self.partial.addr_count)
+
+    def adjust_break(self, btype: BreakType, penalty: float) -> None:
+        """ Change the break type and penalty for this node.
+        """
+        self.btype = btype
+        self.penalty = penalty
 
     def has_tokens(self, end: int, *ttypes: TokenType) -> bool:
         """ Check if there are tokens of the given types ending at the
@@ -211,26 +262,37 @@ class QueryStruct:
         need to be direct neighbours. Thus the query is represented as a
         directed acyclic graph.
 
+        A query also has a direction penalty 'dir_penalty'. This describes
+        the likelyhood if the query should be read from left-to-right or
+        vice versa. A negative 'dir_penalty' should be read as a penalty on
+        right-to-left reading, while a positive value represents a penalty
+        for left-to-right reading. The default value is 0, which is equivalent
+        to having no information about the reading.
+
         When created, a query contains a single node: the start of the
         query. Further nodes can be added by appending to 'nodes'.
     """
 
     def __init__(self, source: List[Phrase]) -> None:
         self.source = source
+        self.dir_penalty = 0.0
         self.nodes: List[QueryNode] = \
-            [QueryNode(BreakType.START, source[0].ptype if source else PhraseType.NONE)]
+            [QueryNode(BREAK_START, source[0].ptype if source else PHRASE_ANY,
+                       0.0, '', '')]
 
     def num_token_slots(self) -> int:
         """ Return the length of the query in vertice steps.
         """
         return len(self.nodes) - 1
 
-    def add_node(self, btype: BreakType, ptype: PhraseType) -> None:
+    def add_node(self, btype: BreakType, ptype: PhraseType,
+                 break_penalty: float = 0.0,
+                 term_lookup: str = '', term_normalized: str = '') -> None:
         """ Append a new break node with the given break type.
             The phrase type denotes the type for any tokens starting
             at the node.
         """
-        self.nodes.append(QueryNode(btype, ptype))
+        self.nodes.append(QueryNode(btype, ptype, break_penalty, term_lookup, term_normalized))
 
     def add_token(self, trange: TokenRange, ttype: TokenType, token: Token) -> None:
         """ Add a token to the query. 'start' and 'end' are the indexes of the
@@ -243,37 +305,63 @@ class QueryStruct:
             be added to, then the token is silently dropped.
         """
         snode = self.nodes[trange.start]
-        full_phrase = snode.btype in (BreakType.START, BreakType.PHRASE)\
-            and self.nodes[trange.end].btype in (BreakType.PHRASE, BreakType.END)
-        if snode.ptype.compatible_with(ttype, full_phrase):
-            tlist = snode.get_tokens(trange.end, ttype)
-            if tlist is None:
-                snode.starting.append(TokenList(trange.end, ttype, [token]))
-            else:
-                tlist.append(token)
+        if ttype == TOKEN_PARTIAL:
+            assert snode.partial is None
+            if _phrase_compatible_with(snode.ptype, TOKEN_PARTIAL, False):
+                snode.partial = token
+        else:
+            full_phrase = snode.btype in (BREAK_START, BREAK_PHRASE)\
+                and self.nodes[trange.end].btype in (BREAK_PHRASE, BREAK_END)
+            if _phrase_compatible_with(snode.ptype, ttype, full_phrase):
+                tlist = snode.get_tokens(trange.end, ttype)
+                if tlist is None:
+                    snode.starting.append(TokenList(trange.end, ttype, [token]))
+                else:
+                    tlist.append(token)
+
+    def compute_direction_penalty(self) -> None:
+        """ Recompute the direction probability from the partial tokens
+            of each node.
+        """
+        n = len(self.nodes) - 1
+        if n == 1 or n >= 50:
+            self.dir_penalty = 0
+        elif n == 2:
+            self.dir_penalty = (self.nodes[1].name_address_ratio()
+                                - self.nodes[0].name_address_ratio()) / 3
+        else:
+            ratios = [n.name_address_ratio() for n in self.nodes[:-1]]
+            self.dir_penalty = (n * sum(i * r for i, r in enumerate(ratios))
+                                - sum(ratios) * n * (n - 1) / 2) / LINFAC[n]
 
     def get_tokens(self, trange: TokenRange, ttype: TokenType) -> List[Token]:
         """ Get the list of tokens of a given type, spanning the given
             nodes. The nodes must exist. If no tokens exist, an
             empty list is returned.
+
+            Cannot be used to get the partial token.
         """
+        assert ttype != TOKEN_PARTIAL
         return self.nodes[trange.start].get_tokens(trange.end, ttype) or []
 
-    def get_partials_list(self, trange: TokenRange) -> List[Token]:
-        """ Create a list of partial tokens between the given nodes.
-            The list is composed of the first token of type PARTIAL
-            going to the subsequent node. Such PARTIAL tokens are
-            assumed to exist.
+    def iter_partials(self, trange: TokenRange) -> Iterator[Token]:
+        """ Iterate over the partial tokens between the given nodes.
+            Missing partials are ignored.
         """
-        return [next(iter(self.get_tokens(TokenRange(i, i+1), TokenType.PARTIAL)))
-                for i in range(trange.start, trange.end)]
+        return (n.partial for n in self.nodes[trange.start:trange.end] if n.partial is not None)
 
-    def iter_token_lists(self) -> Iterator[Tuple[int, QueryNode, TokenList]]:
-        """ Iterator over all token lists in the query.
+    def iter_tokens_by_edge(self) -> Iterator[Tuple[int, int, Dict[TokenType, List[Token]]]]:
+        """ Iterator over all tokens except partial ones grouped by edge.
+
+            Returns the start and end node indexes and a dictionary
+            of list of tokens by token type.
         """
         for i, node in enumerate(self.nodes):
+            by_end: Dict[int, Dict[TokenType, List[Token]]] = defaultdict(dict)
             for tlist in node.starting:
-                yield i, node, tlist
+                by_end[tlist.end][tlist.ttype] = tlist.tokens
+            for end, endlist in by_end.items():
+                yield i, end, endlist
 
     def find_lookup_word_by_id(self, token: int) -> str:
         """ Find the first token with the given token ID and return
@@ -282,8 +370,51 @@ class QueryStruct:
             debugging.
         """
         for node in self.nodes:
+            if node.partial is not None and node.partial.token == token:
+                return f"[P]{node.partial.lookup_word}"
             for tlist in node.starting:
                 for t in tlist.tokens:
                     if t.token == token:
-                        return f"[{tlist.ttype.name[0]}]{t.lookup_word}"
+                        return f"[{tlist.ttype}]{t.lookup_word}"
         return 'None'
+
+    def get_transliterated_query(self) -> str:
+        """ Return a string representation of the transliterated query
+            with the character representation of the different break types.
+
+            For debugging purposes only.
+        """
+        return ''.join(''.join((n.term_lookup, n.btype)) for n in self.nodes)
+
+    def extract_words(self, base_penalty: float = 0.0,
+                      start: int = 0,
+                      endpos: Optional[int] = None) -> Dict[str, List[TokenRange]]:
+        """ Add all combinations of words that can be formed from the terms
+            between the given start and endnode. The terms are joined with
+            spaces for each break. Words can never go across a BREAK_PHRASE.
+
+            The functions returns a dictionary of possible words with their
+            position within the query and a penalty. The penalty is computed
+            from the base_penalty plus the penalty for each node the word
+            crosses.
+        """
+        if endpos is None:
+            endpos = len(self.nodes)
+
+        words: Dict[str, List[TokenRange]] = defaultdict(list)
+
+        for first, first_node in enumerate(self.nodes[start + 1:endpos], start):
+            word = first_node.term_lookup
+            penalty = base_penalty
+            words[word].append(TokenRange(first, first + 1, penalty=penalty))
+            if first_node.btype != BREAK_PHRASE:
+                penalty += first_node.penalty
+                max_last = min(first + 20, endpos)
+                for last, last_node in enumerate(self.nodes[first + 2:max_last], first + 2):
+                    word = ' '.join((word, last_node.term_lookup))
+                    words[word].append(TokenRange(first, last, penalty=penalty))
+                    if last_node.btype == BREAK_PHRASE:
+                        break
+                    penalty += last_node.penalty
+
+        return words

src/nominatim_api/search/token_assignment.py

@@ -24,13 +24,13 @@ class TypedRange:
 
 
 PENALTY_TOKENCHANGE = {
-    qmod.BreakType.START: 0.0,
-    qmod.BreakType.END: 0.0,
-    qmod.BreakType.PHRASE: 0.0,
-    qmod.BreakType.SOFT_PHRASE: 0.0,
-    qmod.BreakType.WORD: 0.1,
-    qmod.BreakType.PART: 0.2,
-    qmod.BreakType.TOKEN: 0.4
+    qmod.BREAK_START: 0.0,
+    qmod.BREAK_END: 0.0,
+    qmod.BREAK_PHRASE: 0.0,
+    qmod.BREAK_SOFT_PHRASE: 0.0,
+    qmod.BREAK_WORD: 0.1,
+    qmod.BREAK_PART: 0.2,
+    qmod.BREAK_TOKEN: 0.4
 }
 
 TypedRangeSeq = List[TypedRange]
@@ -56,17 +56,17 @@ class TokenAssignment:
         """
         out = TokenAssignment()
         for token in ranges:
-            if token.ttype == qmod.TokenType.PARTIAL:
+            if token.ttype == qmod.TOKEN_PARTIAL:
                 out.address.append(token.trange)
-            elif token.ttype == qmod.TokenType.HOUSENUMBER:
+            elif token.ttype == qmod.TOKEN_HOUSENUMBER:
                 out.housenumber = token.trange
-            elif token.ttype == qmod.TokenType.POSTCODE:
+            elif token.ttype == qmod.TOKEN_POSTCODE:
                 out.postcode = token.trange
-            elif token.ttype == qmod.TokenType.COUNTRY:
+            elif token.ttype == qmod.TOKEN_COUNTRY:
                 out.country = token.trange
-            elif token.ttype == qmod.TokenType.NEAR_ITEM:
+            elif token.ttype == qmod.TOKEN_NEAR_ITEM:
                 out.near_item = token.trange
-            elif token.ttype == qmod.TokenType.QUALIFIER:
+            elif token.ttype == qmod.TOKEN_QUALIFIER:
                 out.qualifier = token.trange
         return out
 
@@ -84,7 +84,7 @@ class _TokenSequence:
         self.penalty = penalty
 
     def __str__(self) -> str:
-        seq = ''.join(f'[{r.trange.start} - {r.trange.end}: {r.ttype.name}]' for r in self.seq)
+        seq = ''.join(f'[{r.trange.start} - {r.trange.end}: {r.ttype}]' for r in self.seq)
         return f'{seq} (dir: {self.direction}, penalty: {self.penalty})'
 
     @property
@@ -105,7 +105,7 @@ class _TokenSequence:
         """
         # Country and category must be the final term for left-to-right
         return len(self.seq) > 1 and \
-            self.seq[-1].ttype in (qmod.TokenType.COUNTRY, qmod.TokenType.NEAR_ITEM)
+            self.seq[-1].ttype in (qmod.TOKEN_COUNTRY, qmod.TOKEN_NEAR_ITEM)
 
     def appendable(self, ttype: qmod.TokenType) -> Optional[int]:
         """ Check if the give token type is appendable to the existing sequence.
@@ -114,23 +114,23 @@ class _TokenSequence:
             new direction of the sequence after adding such a type. The
             token is not added.
         """
-        if ttype == qmod.TokenType.WORD:
+        if ttype == qmod.TOKEN_WORD:
             return None
 
         if not self.seq:
             # Append unconditionally to the empty list
-            if ttype == qmod.TokenType.COUNTRY:
+            if ttype == qmod.TOKEN_COUNTRY:
                 return -1
-            if ttype in (qmod.TokenType.HOUSENUMBER, qmod.TokenType.QUALIFIER):
+            if ttype in (qmod.TOKEN_HOUSENUMBER, qmod.TOKEN_QUALIFIER):
                 return 1
             return self.direction
 
         # Name tokens are always acceptable and don't change direction
-        if ttype == qmod.TokenType.PARTIAL:
+        if ttype == qmod.TOKEN_PARTIAL:
             # qualifiers cannot appear in the middle of the query. They need
             # to be near the next phrase.
             if self.direction == -1 \
-               and any(t.ttype == qmod.TokenType.QUALIFIER for t in self.seq[:-1]):
+               and any(t.ttype == qmod.TOKEN_QUALIFIER for t in self.seq[:-1]):
                 return None
             return self.direction
 
@@ -138,54 +138,54 @@ class _TokenSequence:
         if self.has_types(ttype):
             return None
 
-        if ttype == qmod.TokenType.HOUSENUMBER:
+        if ttype == qmod.TOKEN_HOUSENUMBER:
             if self.direction == 1:
-                if len(self.seq) == 1 and self.seq[0].ttype == qmod.TokenType.QUALIFIER:
+                if len(self.seq) == 1 and self.seq[0].ttype == qmod.TOKEN_QUALIFIER:
                     return None
                 if len(self.seq) > 2 \
-                   or self.has_types(qmod.TokenType.POSTCODE, qmod.TokenType.COUNTRY):
+                   or self.has_types(qmod.TOKEN_POSTCODE, qmod.TOKEN_COUNTRY):
                     return None  # direction left-to-right: housenumber must come before anything
             elif (self.direction == -1
-                  or self.has_types(qmod.TokenType.POSTCODE, qmod.TokenType.COUNTRY)):
+                  or self.has_types(qmod.TOKEN_POSTCODE, qmod.TOKEN_COUNTRY)):
                 return -1  # force direction right-to-left if after other terms
 
             return self.direction
 
-        if ttype == qmod.TokenType.POSTCODE:
+        if ttype == qmod.TOKEN_POSTCODE:
             if self.direction == -1:
-                if self.has_types(qmod.TokenType.HOUSENUMBER, qmod.TokenType.QUALIFIER):
+                if self.has_types(qmod.TOKEN_HOUSENUMBER, qmod.TOKEN_QUALIFIER):
                     return None
                 return -1
             if self.direction == 1:
-                return None if self.has_types(qmod.TokenType.COUNTRY) else 1
-            if self.has_types(qmod.TokenType.HOUSENUMBER, qmod.TokenType.QUALIFIER):
+                return None if self.has_types(qmod.TOKEN_COUNTRY) else 1
+            if self.has_types(qmod.TOKEN_HOUSENUMBER, qmod.TOKEN_QUALIFIER):
                 return 1
             return self.direction
 
-        if ttype == qmod.TokenType.COUNTRY:
+        if ttype == qmod.TOKEN_COUNTRY:
             return None if self.direction == -1 else 1
 
-        if ttype == qmod.TokenType.NEAR_ITEM:
+        if ttype == qmod.TOKEN_NEAR_ITEM:
             return self.direction
 
-        if ttype == qmod.TokenType.QUALIFIER:
+        if ttype == qmod.TOKEN_QUALIFIER:
             if self.direction == 1:
                 if (len(self.seq) == 1
-                    and self.seq[0].ttype in (qmod.TokenType.PARTIAL, qmod.TokenType.NEAR_ITEM)) \
+                    and self.seq[0].ttype in (qmod.TOKEN_PARTIAL, qmod.TOKEN_NEAR_ITEM)) \
                    or (len(self.seq) == 2
-                       and self.seq[0].ttype == qmod.TokenType.NEAR_ITEM
-                       and self.seq[1].ttype == qmod.TokenType.PARTIAL):
+                       and self.seq[0].ttype == qmod.TOKEN_NEAR_ITEM
+                       and self.seq[1].ttype == qmod.TOKEN_PARTIAL):
                     return 1
                 return None
             if self.direction == -1:
                 return -1
 
-            tempseq = self.seq[1:] if self.seq[0].ttype == qmod.TokenType.NEAR_ITEM else self.seq
+            tempseq = self.seq[1:] if self.seq[0].ttype == qmod.TOKEN_NEAR_ITEM else self.seq
             if len(tempseq) == 0:
                 return 1
-            if len(tempseq) == 1 and self.seq[0].ttype == qmod.TokenType.HOUSENUMBER:
+            if len(tempseq) == 1 and self.seq[0].ttype == qmod.TOKEN_HOUSENUMBER:
                 return None
-            if len(tempseq) > 1 or self.has_types(qmod.TokenType.POSTCODE, qmod.TokenType.COUNTRY):
+            if len(tempseq) > 1 or self.has_types(qmod.TOKEN_POSTCODE, qmod.TOKEN_COUNTRY):
                 return -1
             return 0
 
@@ -205,7 +205,7 @@ class _TokenSequence:
             new_penalty = 0.0
         else:
             last = self.seq[-1]
-            if btype != qmod.BreakType.PHRASE and last.ttype == ttype:
+            if btype != qmod.BREAK_PHRASE and last.ttype == ttype:
                 # extend the existing range
                 newseq = self.seq[:-1] + [TypedRange(ttype, last.trange.replace_end(end_pos))]
                 new_penalty = 0.0
@@ -240,18 +240,18 @@ class _TokenSequence:
         # housenumbers may not be further than 2 words from the beginning.
         # If there are two words in front, give it a penalty.
         hnrpos = next((i for i, tr in enumerate(self.seq)
-                       if tr.ttype == qmod.TokenType.HOUSENUMBER),
+                       if tr.ttype == qmod.TOKEN_HOUSENUMBER),
                       None)
         if hnrpos is not None:
             if self.direction != -1:
-                priors = sum(1 for t in self.seq[:hnrpos] if t.ttype == qmod.TokenType.PARTIAL)
+                priors = sum(1 for t in self.seq[:hnrpos] if t.ttype == qmod.TOKEN_PARTIAL)
                 if not self._adapt_penalty_from_priors(priors, -1):
                     return False
             if self.direction != 1:
-                priors = sum(1 for t in self.seq[hnrpos+1:] if t.ttype == qmod.TokenType.PARTIAL)
+                priors = sum(1 for t in self.seq[hnrpos+1:] if t.ttype == qmod.TOKEN_PARTIAL)
                 if not self._adapt_penalty_from_priors(priors, 1):
                     return False
-            if any(t.ttype == qmod.TokenType.NEAR_ITEM for t in self.seq):
+            if any(t.ttype == qmod.TOKEN_NEAR_ITEM for t in self.seq):
                 self.penalty += 1.0
 
         return True
@@ -269,10 +269,9 @@ class _TokenSequence:
             # <address>,<postcode> should give preference to address search
             if base.postcode.start == 0:
                 penalty = self.penalty
-                self.direction = -1  # name searches are only possible backwards
             else:
                 penalty = self.penalty + 0.1
-                self.direction = 1  # name searches are only possible forwards
+            penalty += 0.1 * max(0, len(base.address) - 1)
             yield dataclasses.replace(base, penalty=penalty)
 
     def _get_assignments_address_forward(self, base: TokenAssignment,
@@ -282,8 +281,17 @@ class _TokenSequence:
         """
         first = base.address[0]
 
+        # The postcode must come after the name.
+        if base.postcode and base.postcode < first:
+            log().var_dump('skip forward', (base.postcode, first))
+            return
+
+        penalty = self.penalty
+        if not base.country and self.direction == 1 and query.dir_penalty > 0:
+            penalty += query.dir_penalty
+
         log().comment('first word = name')
-        yield dataclasses.replace(base, penalty=self.penalty,
+        yield dataclasses.replace(base, penalty=penalty,
                                   name=first, address=base.address[1:])
 
         # To paraphrase:
@@ -293,17 +301,18 @@ class _TokenSequence:
         #  * the containing phrase is strictly typed
         if (base.housenumber and first.end < base.housenumber.start)\
            or (base.qualifier and base.qualifier > first)\
-           or (query.nodes[first.start].ptype != qmod.PhraseType.NONE):
+           or (query.nodes[first.start].ptype != qmod.PHRASE_ANY):
             return
 
-        penalty = self.penalty
-
         # Penalty for:
         #  * <name>, <street>, <housenumber> , ...
         #  * queries that are comma-separated
         if (base.housenumber and base.housenumber > first) or len(query.source) > 1:
             penalty += 0.25
 
+        if self.direction == 0 and query.dir_penalty > 0:
+            penalty += query.dir_penalty
+
         for i in range(first.start + 1, first.end):
             name, addr = first.split(i)
             log().comment(f'split first word = name ({i - first.start})')
@@ -317,9 +326,18 @@ class _TokenSequence:
         """
         last = base.address[-1]
 
-        if self.direction == -1 or len(base.address) > 1:
+        # The postcode must come before the name for backward direction.
+        if base.postcode and base.postcode > last:
+            log().var_dump('skip backward', (base.postcode, last))
+            return
+
+        penalty = self.penalty
+        if not base.country and self.direction == -1 and query.dir_penalty < 0:
+            penalty -= query.dir_penalty
+
+        if self.direction == -1 or len(base.address) > 1 or base.postcode:
             log().comment('last word = name')
-            yield dataclasses.replace(base, penalty=self.penalty,
+            yield dataclasses.replace(base, penalty=penalty,
                                       name=last, address=base.address[:-1])
 
         # To paraphrase:
@@ -329,15 +347,17 @@ class _TokenSequence:
         #  * the containing phrase is strictly typed
         if (base.housenumber and last.start > base.housenumber.end)\
            or (base.qualifier and base.qualifier < last)\
-           or (query.nodes[last.start].ptype != qmod.PhraseType.NONE):
+           or (query.nodes[last.start].ptype != qmod.PHRASE_ANY):
             return
 
-        penalty = self.penalty
         if base.housenumber and base.housenumber < last:
             penalty += 0.4
         if len(query.source) > 1:
             penalty += 0.25
 
+        if self.direction == 0 and query.dir_penalty < 0:
+            penalty -= query.dir_penalty
+
         for i in range(last.start + 1, last.end):
             addr, name = last.split(i)
             log().comment(f'split last word = name ({i - last.start})')
@@ -370,11 +390,11 @@ class _TokenSequence:
             if base.postcode and base.postcode.start == 0:
                 self.penalty += 0.1
 
-            # Right-to-left reading of the address
+            # Left-to-right reading of the address
             if self.direction != -1:
                 yield from self._get_assignments_address_forward(base, query)
 
-            # Left-to-right reading of the address
+            # Right-to-left reading of the address
             if self.direction != 1:
                 yield from self._get_assignments_address_backward(base, query)
 
@@ -393,18 +413,29 @@ def yield_token_assignments(query: qmod.QueryStruct) -> Iterator[TokenAssignment
         another. It does not include penalties for transitions within a
         type.
     """
-    todo = [_TokenSequence([], direction=0 if query.source[0].ptype == qmod.PhraseType.NONE else 1)]
+    todo = [_TokenSequence([], direction=0 if query.source[0].ptype == qmod.PHRASE_ANY else 1)]
 
     while todo:
         state = todo.pop()
         node = query.nodes[state.end_pos]
 
         for tlist in node.starting:
-            newstate = state.advance(tlist.ttype, tlist.end, node.btype)
-            if newstate is not None:
-                if newstate.end_pos == query.num_token_slots():
-                    if newstate.recheck_sequence():
-                        log().var_dump('Assignment', newstate)
-                        yield from newstate.get_assignments(query)
-                elif not newstate.is_final():
-                    todo.append(newstate)
+            yield from _append_state_to_todo(
+                query, todo,
+                state.advance(tlist.ttype, tlist.end, node.btype))
+
+        if node.partial is not None:
+            yield from _append_state_to_todo(
+                query, todo,
+                state.advance(qmod.TOKEN_PARTIAL, state.end_pos + 1, node.btype))
+
+
+def _append_state_to_todo(query: qmod.QueryStruct, todo: List[_TokenSequence],
+                          newstate: Optional[_TokenSequence]) -> Iterator[TokenAssignment]:
+    if newstate is not None:
+        if newstate.end_pos == query.num_token_slots():
+            if newstate.recheck_sequence():
+                log().var_dump('Assignment', newstate)
+                yield from newstate.get_assignments(query)
+        elif not newstate.is_final():
+            todo.append(newstate)

src/nominatim_api/server/starlette/server.py

@@ -143,7 +143,7 @@ def get_application(project_dir: Path,
 
     log_file = config.LOG_FILE
     if log_file:
-        middleware.append(Middleware(FileLoggingMiddleware, file_name=log_file))
+        middleware.append(Middleware(FileLoggingMiddleware, file_name=log_file))  # type: ignore
 
     exceptions: Dict[Any, Callable[[Request, Exception], Awaitable[Response]]] = {
         TimeoutError: timeout_error,

src/nominatim_api/sql/sqlalchemy_functions.py

@@ -122,15 +122,18 @@ class IsAddressPoint(sa.sql.functions.GenericFunction[Any]):
 
     def __init__(self, table: sa.Table) -> None:
         super().__init__(table.c.rank_address,
-                         table.c.housenumber, table.c.name)
+                         table.c.housenumber, table.c.name, table.c.address)
 
 
 @compiles(IsAddressPoint)
 def default_is_address_point(element: IsAddressPoint,
                              compiler: 'sa.Compiled', **kw: Any) -> str:
-    rank, hnr, name = list(element.clauses)
-    return "(%s = 30 AND (%s IS NOT NULL OR %s ? 'addr:housename'))" % (
+    rank, hnr, name, address = list(element.clauses)
+    return "(%s = 30 AND (%s IS NULL OR NOT %s ? '_inherited')" \
+           " AND (%s IS NOT NULL OR %s ? 'addr:housename'))" % (
                 compiler.process(rank, **kw),
+                compiler.process(address, **kw),
+                compiler.process(address, **kw),
                 compiler.process(hnr, **kw),
                 compiler.process(name, **kw))
 
@@ -138,9 +141,11 @@ def default_is_address_point(element: IsAddressPoint,
 @compiles(IsAddressPoint, 'sqlite')
 def sqlite_is_address_point(element: IsAddressPoint,
                             compiler: 'sa.Compiled', **kw: Any) -> str:
-    rank, hnr, name = list(element.clauses)
-    return "(%s = 30 AND coalesce(%s, json_extract(%s, '$.addr:housename')) IS NOT NULL)" % (
+    rank, hnr, name, address = list(element.clauses)
+    return "(%s = 30 AND json_extract(%s, '$._inherited') IS NULL" \
+           " AND coalesce(%s, json_extract(%s, '$.addr:housename')) IS NOT NULL)" % (
                 compiler.process(rank, **kw),
+                compiler.process(address, **kw),
                 compiler.process(hnr, **kw),
                 compiler.process(name, **kw))
 

src/nominatim_api/sql/sqlalchemy_types/geometry.py

@@ -173,7 +173,7 @@ class Geometry(types.UserDefinedType):  # type: ignore[type-arg]
     def __init__(self, subtype: str = 'Geometry'):
         self.subtype = subtype
 
-    def get_col_spec(self) -> str:
+    def get_col_spec(self, **_: Any) -> str:
         return f'GEOMETRY({self.subtype}, 4326)'
 
     def bind_processor(self, dialect: 'sa.Dialect') -> Callable[[Any], str]:

src/nominatim_api/sql/sqlalchemy_types/json.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Common json type for different dialects.
@@ -24,6 +24,6 @@ class Json(sa.types.TypeDecorator[Any]):
 
     def load_dialect_impl(self, dialect: SaDialect) -> sa.types.TypeEngine[Any]:
         if dialect.name == 'postgresql':
-            return JSONB(none_as_null=True)  # type: ignore[no-untyped-call]
+            return JSONB(none_as_null=True)
 
         return sqlite_json(none_as_null=True)

src/nominatim_api/types.py

@@ -144,7 +144,7 @@ class Point(NamedTuple):
         except ValueError as exc:
             raise UsageError('Point parameter needs to be numbers.') from exc
 
-        if x < -180.0 or x > 180.0 or y < -90.0 or y > 90.0:
+        if not -180 <= x <= 180 or not -90 <= y <= 90.0:
             raise UsageError('Point coordinates invalid.')
 
         return Point(x, y)

src/nominatim_api/v1/classtypes.py

@@ -25,8 +25,8 @@ def get_label_tag(category: Tuple[str, str], extratags: Optional[Mapping[str, st
     elif rank < 26 and extratags and 'linked_place' in extratags:
         label = extratags['linked_place']
     elif category == ('boundary', 'administrative'):
-        label = ADMIN_LABELS.get((country or '', int(rank/2)))\
-                or ADMIN_LABELS.get(('', int(rank/2)))\
+        label = ADMIN_LABELS.get((country or '', rank // 2))\
+                or ADMIN_LABELS.get(('', rank // 2))\
                 or 'Administrative'
     elif category[1] == 'postal_code':
         label = 'postcode'

src/nominatim_api/v1/format_json.py

@@ -84,8 +84,9 @@ def format_base_json(results: Union[ReverseResults, SearchResults],
 
         _write_osm_id(out, result.osm_object)
 
-        out.keyval('lat', f"{result.centroid.lat}")\
-           .keyval('lon', f"{result.centroid.lon}")\
+        # lat and lon must be string values
+        out.keyval('lat', f"{result.centroid.lat:0.7f}")\
+           .keyval('lon', f"{result.centroid.lon:0.7f}")\
            .keyval(class_label, result.category[0])\
            .keyval('type', result.category[1])\
            .keyval('place_rank', result.rank_search)\
@@ -112,6 +113,7 @@ def format_base_json(results: Union[ReverseResults, SearchResults],
         if options.get('namedetails', False):
             out.keyval('namedetails', result.names)
 
+        # must be string values
         bbox = cl.bbox_from_result(result)
         out.key('boundingbox').start_array()\
            .value(f"{bbox.minlat:0.7f}").next()\
@@ -249,6 +251,9 @@ def format_base_geocodejson(results: Union[ReverseResults, SearchResults],
                         out.keyval(f"level{line.admin_level}", line.local_name)
             out.end_object().next()
 
+        if options.get('extratags', False):
+            out.keyval('extra', result.extratags)
+
         out.end_object().next().end_object().next()
 
         out.key('geometry').raw(result.geometry.get('geojson')

src/nominatim_api/v1/format_xml.py

@@ -90,7 +90,7 @@ def format_base_xml(results: Union[ReverseResults, SearchResults],
         result will be output, otherwise a list.
     """
     root = ET.Element(xml_root_tag)
-    root.set('timestamp', dt.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S +00:00'))
+    root.set('timestamp', dt.datetime.now(dt.timezone.utc).strftime('%a, %d %b %Y %H:%M:%S +00:00'))
     root.set('attribution', cl.OSM_ATTRIBUTION)
     for k, v in xml_extra_info.items():
         root.set(k, v)

src/nominatim_api/version.py

@@ -8,4 +8,4 @@
 Version information for the Nominatim API.
 """
 
-NOMINATIM_API_VERSION = '5.0.0'
+NOMINATIM_API_VERSION = '5.1.0'

src/nominatim_db/cli.py

@@ -2,16 +2,15 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Command-line interface to the Nominatim functions for import, update,
 database administration and querying.
 """
-from typing import Optional, Any
+from typing import Optional, List, Mapping
 import importlib
 import logging
-import os
 import sys
 import argparse
 import asyncio
@@ -81,13 +80,14 @@ class CommandlineParser:
         parser.set_defaults(command=cmd)
         cmd.add_args(parser)
 
-    def run(self, **kwargs: Any) -> int:
+    def run(self, cli_args: Optional[List[str]],
+            environ: Optional[Mapping[str, str]]) -> int:
         """ Parse the command line arguments of the program and execute the
             appropriate subcommand.
         """
         args = NominatimArgs()
         try:
-            self.parser.parse_args(args=kwargs.get('cli_args'), namespace=args)
+            self.parser.parse_args(args=cli_args, namespace=args)
         except SystemExit:
             return 1
 
@@ -101,23 +101,19 @@ class CommandlineParser:
 
         args.project_dir = Path(args.project_dir).resolve()
 
-        if 'cli_args' not in kwargs:
+        if cli_args is None:
             logging.basicConfig(stream=sys.stderr,
                                 format='%(asctime)s: %(message)s',
                                 datefmt='%Y-%m-%d %H:%M:%S',
                                 level=max(4 - args.verbose, 1) * 10)
 
-        args.config = Configuration(args.project_dir,
-                                    environ=kwargs.get('environ', os.environ))
-        args.config.set_libdirs(osm2pgsql=kwargs['osm2pgsql_path'])
+        args.config = Configuration(args.project_dir, environ=environ)
 
         log = logging.getLogger()
         log.warning('Using project directory: %s', str(args.project_dir))
 
         try:
-            ret = args.command.run(args)
-
-            return ret
+            return args.command.run(args)
         except UsageError as exception:
             if log.isEnabledFor(logging.DEBUG):
                 raise  # use Python's exception printing
@@ -233,9 +229,16 @@ def get_set_parser() -> CommandlineParser:
     return parser
 
 
-def nominatim(**kwargs: Any) -> int:
+def nominatim(cli_args: Optional[List[str]] = None,
+              environ: Optional[Mapping[str, str]] = None) -> int:
     """\
     Command-line tools for importing, updating, administrating and
     querying the Nominatim database.
+
+    'cli_args' is a list of parameters for the command to run. If not given,
+    sys.args will be used.
+
+    'environ' is the dictionary of environment variables containing the
+    Nominatim configuration. When None, the os.environ is inherited.
     """
-    return get_set_parser().run(**kwargs)
+    return get_set_parser().run(cli_args=cli_args, environ=environ)

src/nominatim_db/clicmd/args.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Provides custom functions over command-line arguments.
@@ -136,6 +136,7 @@ class NominatimArgs:
     import_from_wiki: bool
     import_from_csv: Optional[str]
     no_replace: bool
+    min: int
 
     # Arguments to all query functions
     format: str
@@ -186,7 +187,7 @@ class NominatimArgs:
             from the command line arguments. The resulting dict can be
             further customized and then used in `run_osm2pgsql()`.
         """
-        return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.config.lib_dir.osm2pgsql,
+        return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY,
                     osm2pgsql_cache=self.osm2pgsql_cache or default_cache,
                     osm2pgsql_style=self.config.get_import_style_file(),
                     osm2pgsql_style_path=self.config.lib_dir.lua,

src/nominatim_db/clicmd/special_phrases.py

@@ -58,6 +58,8 @@ class ImportSpecialPhrases:
                            help='Import special phrases from a CSV file')
         group.add_argument('--no-replace', action='store_true',
                            help='Keep the old phrases and only add the new ones')
+        group.add_argument('--min', type=int, default=0,
+                           help='Restrict special phrases by minimum occurance')
 
     def run(self, args: NominatimArgs) -> int:
 
@@ -82,7 +84,9 @@ class ImportSpecialPhrases:
 
         tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
         should_replace = not args.no_replace
+        min = args.min
+
         with connect(args.config.get_libpq_dsn()) as db_connection:
             SPImporter(
                 args.config, db_connection, loader
-            ).import_phrases(tokenizer, should_replace)
+            ).import_phrases(tokenizer, should_replace, min)

src/nominatim_db/config.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Nominatim configuration accessor.
@@ -73,7 +73,6 @@ class Configuration:
             self.project_dir = None
 
         class _LibDirs:
-            osm2pgsql: Path
             sql = paths.SQLLIB_DIR
             lua = paths.LUALIB_DIR
             data = paths.DATA_DIR

src/nominatim_db/db/connection.py

@@ -102,10 +102,10 @@ def server_version_tuple(conn: Connection) -> Tuple[int, int]:
         Converts correctly for pre-10 and post-10 PostgreSQL versions.
     """
     version = conn.info.server_version
-    if version < 100000:
-        return (int(version / 10000), int((version % 10000) / 100))
-
-    return (int(version / 10000), version % 10000)
+    major, minor = divmod(version, 10000)
+    if major < 10:
+        minor //= 100
+    return major, minor
 
 
 def postgis_version_tuple(conn: Connection) -> Tuple[int, int]:

src/nominatim_db/indexer/progress.py

@@ -50,8 +50,8 @@ class ProgressLogger:
         places_per_sec = self.done_places / done_time
         eta = (self.total_places - self.done_places) / places_per_sec
 
-        LOG.warning("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f",
-                    self.done_places, int(done_time),
+        LOG.warning("Done %d in %.0f @ %.3f per second - %s ETA (seconds): %.2f",
+                    self.done_places, done_time,
                     places_per_sec, self.name, eta)
 
         self.next_info += int(places_per_sec) * self.log_interval
@@ -68,8 +68,8 @@ class ProgressLogger:
             diff_seconds = (rank_end_time - self.rank_start_time).total_seconds()
             places_per_sec = self.done_places / diff_seconds
 
-        LOG.warning("Done %d/%d in %d @ %.3f per second - FINISHED %s\n",
-                    self.done_places, self.total_places, int(diff_seconds),
+        LOG.warning("Done %d/%d in %.0f @ %.3f per second - FINISHED %s\n",
+                    self.done_places, self.total_places, diff_seconds,
                     places_per_sec, self.name)
 
         return self.done_places

src/nominatim_db/tokenizer/base.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Abstract class definitions for tokenizers. These base classes are here
@@ -10,7 +10,6 @@ mainly for documentation purposes.
 """
 from abc import ABC, abstractmethod
 from typing import List, Tuple, Dict, Any, Optional, Iterable
-from pathlib import Path
 
 from ..typing import Protocol
 from ..config import Configuration
@@ -38,7 +37,7 @@ class AbstractAnalyzer(ABC):
         """
 
     @abstractmethod
-    def get_word_token_info(self, words: List[str]) -> List[Tuple[str, str, int]]:
+    def get_word_token_info(self, words: List[str]) -> List[Tuple[str, str, Optional[int]]]:
         """ Return token information for the given list of words.
 
             The function is used for testing and debugging only
@@ -232,6 +231,6 @@ class TokenizerModule(Protocol):
         own tokenizer.
     """
 
-    def create(self, dsn: str, data_dir: Path) -> AbstractTokenizer:
+    def create(self, dsn: str) -> AbstractTokenizer:
         """ Factory for new tokenizers.
         """

src/nominatim_db/tokenizer/factory.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Functions for creating a tokenizer or initialising the right one for an
@@ -52,19 +52,10 @@ def create_tokenizer(config: Configuration, init_db: bool = True,
     if module_name is None:
         module_name = config.TOKENIZER
 
-    # Create the directory for the tokenizer data
-    assert config.project_dir is not None
-    basedir = config.project_dir / 'tokenizer'
-    if not basedir.exists():
-        basedir.mkdir()
-    elif not basedir.is_dir():
-        LOG.fatal("Tokenizer directory '%s' cannot be created.", basedir)
-        raise UsageError("Tokenizer setup failed.")
-
     # Import and initialize the tokenizer.
     tokenizer_module = _import_tokenizer(module_name)
 
-    tokenizer = tokenizer_module.create(config.get_libpq_dsn(), basedir)
+    tokenizer = tokenizer_module.create(config.get_libpq_dsn())
     tokenizer.init_new_db(config, init_db=init_db)
 
     with connect(config.get_libpq_dsn()) as conn:
@@ -79,12 +70,6 @@ def get_tokenizer_for_db(config: Configuration) -> AbstractTokenizer:
         The function looks up the appropriate tokenizer in the database
         and initialises it.
     """
-    assert config.project_dir is not None
-    basedir = config.project_dir / 'tokenizer'
-    if not basedir.is_dir():
-        # Directory will be repopulated by tokenizer below.
-        basedir.mkdir()
-
     with connect(config.get_libpq_dsn()) as conn:
         name = properties.get_property(conn, 'tokenizer')
 
@@ -94,7 +79,7 @@ def get_tokenizer_for_db(config: Configuration) -> AbstractTokenizer:
 
     tokenizer_module = _import_tokenizer(name)
 
-    tokenizer = tokenizer_module.create(config.get_libpq_dsn(), basedir)
+    tokenizer = tokenizer_module.create(config.get_libpq_dsn())
     tokenizer.init_from_project(config)
 
     return tokenizer

src/nominatim_db/tokenizer/icu_tokenizer.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Tokenizer implementing normalisation as used before Nominatim 4 but using
@@ -12,7 +12,6 @@ from typing import Optional, Sequence, List, Tuple, Mapping, Any, cast, \
                    Dict, Set, Iterable
 import itertools
 import logging
-from pathlib import Path
 
 from psycopg.types.json import Jsonb
 from psycopg import sql as pysql
@@ -38,10 +37,10 @@ WORD_TYPES = (('country_names', 'C'),
               ('housenumbers', 'H'))
 
 
-def create(dsn: str, data_dir: Path) -> 'ICUTokenizer':
+def create(dsn: str) -> 'ICUTokenizer':
     """ Create a new instance of the tokenizer provided by this module.
     """
-    return ICUTokenizer(dsn, data_dir)
+    return ICUTokenizer(dsn)
 
 
 class ICUTokenizer(AbstractTokenizer):
@@ -50,9 +49,8 @@ class ICUTokenizer(AbstractTokenizer):
         normalization routines in Nominatim 3.
     """
 
-    def __init__(self, dsn: str, data_dir: Path) -> None:
+    def __init__(self, dsn: str) -> None:
         self.dsn = dsn
-        self.data_dir = data_dir
         self.loader: Optional[ICURuleLoader] = None
 
     def init_new_db(self, config: Configuration, init_db: bool = True) -> None:
@@ -121,10 +119,10 @@ class ICUTokenizer(AbstractTokenizer):
                            SELECT unnest(nameaddress_vector) as id, count(*)
                                  FROM search_name GROUP BY id)
                   SELECT coalesce(a.id, w.id) as id,
-                         (CASE WHEN w.count is null THEN '{}'::JSONB
+                         (CASE WHEN w.count is null or w.count <= 1 THEN '{}'::JSONB
                               ELSE jsonb_build_object('count', w.count) END
                           ||
-                          CASE WHEN a.count is null THEN '{}'::JSONB
+                          CASE WHEN a.count is null or a.count <= 1 THEN '{}'::JSONB
                               ELSE jsonb_build_object('addr_count', a.count) END) as info
                   FROM word_freq w FULL JOIN addr_freq a ON a.id = w.id;
                   """)
@@ -134,9 +132,10 @@ class ICUTokenizer(AbstractTokenizer):
                 drop_tables(conn, 'tmp_word')
                 cur.execute("""CREATE TABLE tmp_word AS
                                 SELECT word_id, word_token, type, word,
-                                       (CASE WHEN wf.info is null THEN word.info
-                                        ELSE coalesce(word.info, '{}'::jsonb) || wf.info
-                                        END) as info
+                                       coalesce(word.info, '{}'::jsonb)
+                                       - 'count' - 'addr_count' ||
+                                       coalesce(wf.info, '{}'::jsonb)
+                                       as info
                                 FROM word LEFT JOIN word_frequencies wf
                                      ON word.word_id = wf.id
                             """)
@@ -339,7 +338,7 @@ class ICUNameAnalyzer(AbstractAnalyzer):
         """
         return cast(str, self.token_analysis.normalizer.transliterate(name)).strip()
 
-    def get_word_token_info(self, words: Sequence[str]) -> List[Tuple[str, str, int]]:
+    def get_word_token_info(self, words: Sequence[str]) -> List[Tuple[str, str, Optional[int]]]:
         """ Return token information for the given list of words.
             If a word starts with # it is assumed to be a full name
             otherwise is a partial name.
@@ -363,11 +362,11 @@ class ICUNameAnalyzer(AbstractAnalyzer):
             cur.execute("""SELECT word_token, word_id
                             FROM word WHERE word_token = ANY(%s) and type = 'W'
                         """, (list(full_tokens.values()),))
-            full_ids = {r[0]: r[1] for r in cur}
+            full_ids = {r[0]: cast(int, r[1]) for r in cur}
             cur.execute("""SELECT word_token, word_id
                             FROM word WHERE word_token = ANY(%s) and type = 'w'""",
                         (list(partial_tokens.values()),))
-            part_ids = {r[0]: r[1] for r in cur}
+            part_ids = {r[0]: cast(int, r[1]) for r in cur}
 
         return [(k, v, full_ids.get(v, None)) for k, v in full_tokens.items()] \
             + [(k, v, part_ids.get(v, None)) for k, v in partial_tokens.items()]
@@ -381,76 +380,15 @@ class ICUNameAnalyzer(AbstractAnalyzer):
         return postcode.strip().upper()
 
     def update_postcodes_from_db(self) -> None:
-        """ Update postcode tokens in the word table from the location_postcode
-            table.
+        """ Postcode update.
+
+            Removes all postcodes from the word table because they are not
+            needed. Postcodes are recognised by pattern.
         """
         assert self.conn is not None
-        analyzer = self.token_analysis.analysis.get('@postcode')
 
         with self.conn.cursor() as cur:
-            # First get all postcode names currently in the word table.
-            cur.execute("SELECT DISTINCT word FROM word WHERE type = 'P'")
-            word_entries = set((entry[0] for entry in cur))
-
-            # Then compute the required postcode names from the postcode table.
-            needed_entries = set()
-            cur.execute("SELECT country_code, postcode FROM location_postcode")
-            for cc, postcode in cur:
-                info = PlaceInfo({'country_code': cc,
-                                  'class': 'place', 'type': 'postcode',
-                                  'address': {'postcode': postcode}})
-                address = self.sanitizer.process_names(info)[1]
-                for place in address:
-                    if place.kind == 'postcode':
-                        if analyzer is None:
-                            postcode_name = place.name.strip().upper()
-                            variant_base = None
-                        else:
-                            postcode_name = analyzer.get_canonical_id(place)
-                            variant_base = place.get_attr("variant")
-
-                        if variant_base:
-                            needed_entries.add(f'{postcode_name}@{variant_base}')
-                        else:
-                            needed_entries.add(postcode_name)
-                        break
-
-        # Now update the word table.
-        self._delete_unused_postcode_words(word_entries - needed_entries)
-        self._add_missing_postcode_words(needed_entries - word_entries)
-
-    def _delete_unused_postcode_words(self, tokens: Iterable[str]) -> None:
-        assert self.conn is not None
-        if tokens:
-            with self.conn.cursor() as cur:
-                cur.execute("DELETE FROM word WHERE type = 'P' and word = any(%s)",
-                            (list(tokens), ))
-
-    def _add_missing_postcode_words(self, tokens: Iterable[str]) -> None:
-        assert self.conn is not None
-        if not tokens:
-            return
-
-        analyzer = self.token_analysis.analysis.get('@postcode')
-        terms = []
-
-        for postcode_name in tokens:
-            if '@' in postcode_name:
-                term, variant = postcode_name.split('@', 2)
-                term = self._search_normalized(term)
-                if analyzer is None:
-                    variants = [term]
-                else:
-                    variants = analyzer.compute_variants(variant)
-                    if term not in variants:
-                        variants.append(term)
-            else:
-                variants = [self._search_normalized(postcode_name)]
-            terms.append((postcode_name, variants))
-
-        if terms:
-            with self.conn.cursor() as cur:
-                cur.executemany("""SELECT create_postcode_word(%s, %s)""", terms)
+            cur.execute("DELETE FROM word WHERE type = 'P'")
 
     def update_special_phrases(self, phrases: Iterable[Tuple[str, str, str, str]],
                                should_replace: bool) -> None:
@@ -645,10 +583,14 @@ class ICUNameAnalyzer(AbstractAnalyzer):
             if word_id:
                 result = self._cache.housenumbers.get(word_id, result)
                 if result[0] is None:
-                    variants = analyzer.compute_variants(word_id)
+                    varout = analyzer.compute_variants(word_id)
+                    if isinstance(varout, tuple):
+                        variants = varout[0]
+                    else:
+                        variants = varout
                     if variants:
                         hid = execute_scalar(self.conn, "SELECT create_analyzed_hnr_id(%s, %s)",
-                                             (word_id, list(variants)))
+                                             (word_id, variants))
                         result = hid, variants[0]
                         self._cache.housenumbers[word_id] = result
 
@@ -693,13 +635,17 @@ class ICUNameAnalyzer(AbstractAnalyzer):
 
             full, part = self._cache.names.get(token_id, (None, None))
             if full is None:
-                variants = analyzer.compute_variants(word_id)
+                varset = analyzer.compute_variants(word_id)
+                if isinstance(varset, tuple):
+                    variants, lookups = varset
+                else:
+                    variants, lookups = varset, None
                 if not variants:
                     continue
 
                 with self.conn.cursor() as cur:
-                    cur.execute("SELECT * FROM getorcreate_full_word(%s, %s)",
-                                (token_id, variants))
+                    cur.execute("SELECT * FROM getorcreate_full_word(%s, %s, %s)",
+                                (token_id, variants, lookups))
                     full, part = cast(Tuple[int, List[int]], cur.fetchone())
 
                 self._cache.names[token_id] = (full, part)
@@ -718,32 +664,9 @@ class ICUNameAnalyzer(AbstractAnalyzer):
         analyzer = self.token_analysis.analysis.get('@postcode')
 
         if analyzer is None:
-            postcode_name = item.name.strip().upper()
-            variant_base = None
+            return item.name.strip().upper()
         else:
-            postcode_name = analyzer.get_canonical_id(item)
-            variant_base = item.get_attr("variant")
-
-        if variant_base:
-            postcode = f'{postcode_name}@{variant_base}'
-        else:
-            postcode = postcode_name
-
-        if postcode not in self._cache.postcodes:
-            term = self._search_normalized(postcode_name)
-            if not term:
-                return None
-
-            variants = {term}
-            if analyzer is not None and variant_base:
-                variants.update(analyzer.compute_variants(variant_base))
-
-            with self.conn.cursor() as cur:
-                cur.execute("SELECT create_postcode_word(%s, %s)",
-                            (postcode, list(variants)))
-            self._cache.postcodes.add(postcode)
-
-        return postcode_name
+            return analyzer.get_canonical_id(item)
 
 
 class _TokenInfo:
@@ -836,5 +759,4 @@ class _TokenCache:
         self.names: Dict[str, Tuple[int, List[int]]] = {}
         self.partials: Dict[str, int] = {}
         self.fulls: Dict[str, List[int]] = {}
-        self.postcodes: Set[str] = set()
         self.housenumbers: Dict[str, Tuple[Optional[int], Optional[str]]] = {}

src/nominatim_db/tokenizer/token_analysis/base.py

@@ -7,7 +7,7 @@
 """
 Common data types and protocols for analysers.
 """
-from typing import Mapping, List, Any
+from typing import Mapping, List, Any, Union, Tuple
 
 from ...typing import Protocol
 from ...data.place_name import PlaceName
@@ -33,7 +33,7 @@ class Analyzer(Protocol):
                     for example because the character set in use does not match.
         """
 
-    def compute_variants(self, canonical_id: str) -> List[str]:
+    def compute_variants(self, canonical_id: str) -> Union[List[str], Tuple[List[str], List[str]]]:
         """ Compute the transliterated spelling variants for the given
             canonical ID.
 

src/nominatim_db/tokenizer/token_analysis/generic.py

@@ -2,20 +2,19 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Generic processor for names that creates abbreviation variants.
 """
-from typing import Mapping, Dict, Any, Iterable, Iterator, Optional, List, cast
+from typing import Mapping, Dict, Any, Iterable, Optional, List, cast, Tuple
 import itertools
 
-import datrie
-
 from ...errors import UsageError
 from ...data.place_name import PlaceName
 from .config_variants import get_variant_config
 from .generic_mutation import MutationVariantGenerator
+from .simple_trie import SimpleTrie
 
 # Configuration section
 
@@ -25,8 +24,7 @@ def configure(rules: Mapping[str, Any], normalizer: Any, _: Any) -> Dict[str, An
     """
     config: Dict[str, Any] = {}
 
-    config['replacements'], config['chars'] = get_variant_config(rules.get('variants'),
-                                                                 normalizer)
+    config['replacements'], _ = get_variant_config(rules.get('variants'), normalizer)
     config['variant_only'] = rules.get('mode', '') == 'variant-only'
 
     # parse mutation rules
@@ -68,12 +66,8 @@ class GenericTokenAnalysis:
         self.variant_only = config['variant_only']
 
         # Set up datrie
-        if config['replacements']:
-            self.replacements = datrie.Trie(config['chars'])
-            for src, repllist in config['replacements']:
-                self.replacements[src] = repllist
-        else:
-            self.replacements = None
+        self.replacements: Optional[SimpleTrie[List[str]]] = \
+            SimpleTrie(config['replacements']) if config['replacements'] else None
 
         # set up mutation rules
         self.mutations = [MutationVariantGenerator(*cfg) for cfg in config['mutations']]
@@ -84,7 +78,7 @@ class GenericTokenAnalysis:
         """
         return cast(str, self.norm.transliterate(name.name)).strip()
 
-    def compute_variants(self, norm_name: str) -> List[str]:
+    def compute_variants(self, norm_name: str) -> Tuple[List[str], List[str]]:
         """ Compute the spelling variants for the given normalized name
             and transliterate the result.
         """
@@ -93,18 +87,20 @@ class GenericTokenAnalysis:
         for mutation in self.mutations:
             variants = mutation.generate(variants)
 
-        return [name for name in self._transliterate_unique_list(norm_name, variants) if name]
-
-    def _transliterate_unique_list(self, norm_name: str,
-                                   iterable: Iterable[str]) -> Iterator[Optional[str]]:
-        seen = set()
+        varset = set(map(str.strip, variants))
         if self.variant_only:
-            seen.add(norm_name)
+            varset.discard(norm_name)
 
-        for variant in map(str.strip, iterable):
-            if variant not in seen:
-                seen.add(variant)
-                yield self.to_ascii.transliterate(variant).strip()
+        trans = []
+        norm = []
+
+        for var in varset:
+            t = self.to_ascii.transliterate(var).strip()
+            if t:
+                trans.append(t)
+                norm.append(var)
+
+        return trans, norm
 
     def _generate_word_variants(self, norm_name: str) -> Iterable[str]:
         baseform = '^ ' + norm_name + ' ^'
@@ -116,10 +112,10 @@ class GenericTokenAnalysis:
             pos = 0
             force_space = False
             while pos < baselen:
-                full, repl = self.replacements.longest_prefix_item(baseform[pos:],
-                                                                   (None, None))
-                if full is not None:
-                    done = baseform[startpos:pos]
+                frm = pos
+                repl, pos = self.replacements.longest_prefix(baseform, pos)
+                if repl is not None:
+                    done = baseform[startpos:frm]
                     partials = [v + done + r
                                 for v, r in itertools.product(partials, repl)
                                 if not force_space or r.startswith(' ')]
@@ -128,11 +124,10 @@ class GenericTokenAnalysis:
                         # to be helpful. Only use the original term.
                         startpos = 0
                         break
-                    startpos = pos + len(full)
-                    if full[-1] == ' ':
-                        startpos -= 1
+                    if baseform[pos - 1] == ' ':
+                        pos -= 1
                         force_space = True
-                    pos = startpos
+                    startpos = pos
                 else:
                     pos += 1
                     force_space = False

src/nominatim_db/tokenizer/token_analysis/simple_trie.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# This file is part of Nominatim. (https://nominatim.org)
+#
+# Copyright (C) 2025 by the Nominatim developer community.
+# For a full list of authors see the git log.
+"""
+Simple dict-based implementation of a trie structure.
+"""
+from typing import TypeVar, Generic, Tuple, Optional, List, Dict
+from collections import defaultdict
+
+T = TypeVar('T')
+
+
+class SimpleTrie(Generic[T]):
+    """ A simple read-only trie structure.
+        This structure supports examply one lookup operation,
+        which is longest-prefix lookup.
+    """
+
+    def __init__(self, data: Optional[List[Tuple[str, T]]] = None) -> None:
+        self._tree: Dict[str, 'SimpleTrie[T]'] = defaultdict(SimpleTrie[T])
+        self._value: Optional[T] = None
+        self._prefix = ''
+
+        if data:
+            for key, value in data:
+                self._add(key, 0, value)
+
+            self._make_compact()
+
+    def _add(self, word: str, pos: int, value: T) -> None:
+        """ (Internal) Add a sub-word to the trie.
+            The word is added from index 'pos'. If the sub-word to add
+            is empty, then the trie saves the given value.
+        """
+        if pos < len(word):
+            self._tree[word[pos]]._add(word, pos + 1, value)
+        else:
+            self._value = value
+
+    def _make_compact(self) -> None:
+        """ (Internal) Compress tree where there is exactly one subtree
+            and no value.
+
+            Compression works recursively starting at the leaf.
+        """
+        for t in self._tree.values():
+            t._make_compact()
+
+        if len(self._tree) == 1 and self._value is None:
+            assert not self._prefix
+            for k, v in self._tree.items():
+                self._prefix = k + v._prefix
+                self._tree = v._tree
+                self._value = v._value
+
+    def longest_prefix(self, word: str, start: int = 0) -> Tuple[Optional[T], int]:
+        """ Return the longest prefix match for the given word starting at
+            the position 'start'.
+
+            The function returns a tuple with the value for the longest match and
+            the position of the word after the match. If no match was found at
+            all, the function returns (None, start).
+        """
+        cur = self
+        pos = start
+        result: Tuple[Optional[T], int] = None, start
+
+        while True:
+            if cur._prefix:
+                if not word.startswith(cur._prefix, pos):
+                    return result
+                pos += len(cur._prefix)
+
+            if cur._value:
+                result = cur._value, pos
+
+            if pos >= len(word) or word[pos] not in cur._tree:
+                return result
+
+            cur = cur._tree[word[pos]]
+            pos += 1

src/nominatim_db/tools/database_import.py

@@ -127,7 +127,7 @@ def import_osm_data(osm_files: Union[Path, Sequence[Path]],
                 fsize += os.stat(str(fname)).st_size
         else:
             fsize = os.stat(str(osm_files)).st_size
-        options['osm2pgsql_cache'] = int(min((mem.available + mem.cached) * 0.75,
+        options['osm2pgsql_cache'] = int(min((mem.available + getattr(mem, 'cached', 0)) * 0.75,
                                              fsize * 2) / 1024 / 1024) + 1
 
     run_osm2pgsql(options)

src/nominatim_db/tools/exec_utils.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Helper functions for executing external programs.
@@ -85,7 +85,7 @@ def _mk_tablespace_options(ttype: str, options: Mapping[str, Any]) -> List[str]:
 
 
 def _find_osm2pgsql_cmd(cmdline: Optional[str]) -> str:
-    if cmdline is not None:
+    if cmdline:
         return cmdline
 
     in_path = shutil.which('osm2pgsql')

src/nominatim_db/tools/postcodes.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Functions for importing, updating and otherwise maintaining the table
@@ -64,11 +64,15 @@ class _PostcodeCollector:
             if normalized:
                 self.collected[normalized] += (x, y)
 
-    def commit(self, conn: Connection, analyzer: AbstractAnalyzer, project_dir: Path) -> None:
-        """ Update postcodes for the country from the postcodes selected so far
-            as well as any externally supplied postcodes.
+    def commit(self, conn: Connection, analyzer: AbstractAnalyzer,
+               project_dir: Optional[Path]) -> None:
+        """ Update postcodes for the country from the postcodes selected so far.
+
+            When 'project_dir' is set, then any postcode files found in this
+            directory are taken into account as well.
         """
-        self._update_from_external(analyzer, project_dir)
+        if project_dir is not None:
+            self._update_from_external(analyzer, project_dir)
         to_add, to_delete, to_update = self._compute_changes(conn)
 
         LOG.info("Processing country '%s' (%s added, %s deleted, %s updated).",
@@ -170,7 +174,7 @@ class _PostcodeCollector:
         return None
 
 
-def update_postcodes(dsn: str, project_dir: Path, tokenizer: AbstractTokenizer) -> None:
+def update_postcodes(dsn: str, project_dir: Optional[Path], tokenizer: AbstractTokenizer) -> None:
     """ Update the table of artificial postcodes.
 
         Computes artificial postcode centroids from the placex table,

src/nominatim_db/tools/special_phrases/sp_importer.py

@@ -16,7 +16,6 @@
 from typing import Iterable, Tuple, Mapping, Sequence, Optional, Set
 import logging
 import re
-
 from psycopg.sql import Identifier, SQL
 
 from ...typing import Protocol
@@ -65,7 +64,32 @@ class SPImporter():
         # special phrases class/type on the wiki.
         self.table_phrases_to_delete: Set[str] = set()
 
-    def import_phrases(self, tokenizer: AbstractTokenizer, should_replace: bool) -> None:
+    def get_classtype_pairs(self, min: int = 0) -> Set[Tuple[str, str]]:
+        """
+            Returns list of allowed special phrases from the database,
+            restricting to a list of combinations of classes and types
+            which occur equal to or more than a specified amount of times.
+
+            Default value for this is 0, which allows everything in database.
+        """
+        db_combinations = set()
+
+        query = f"""
+        SELECT class AS CLS, type AS typ
+        FROM placex
+        GROUP BY class, type
+        HAVING COUNT(*) >= {min}
+        """
+
+        with self.db_connection.cursor() as db_cursor:
+            db_cursor.execute(SQL(query))
+            for row in db_cursor:
+                db_combinations.add((row[0], row[1]))
+
+        return db_combinations
+
+    def import_phrases(self, tokenizer: AbstractTokenizer, should_replace: bool,
+                       min: int = 0) -> None:
         """
             Iterate through all SpecialPhrases extracted from the
             loader and import them into the database.
@@ -85,9 +109,10 @@ class SPImporter():
             if result:
                 class_type_pairs.add(result)
 
-        self._create_classtype_table_and_indexes(class_type_pairs)
+        self._create_classtype_table_and_indexes(class_type_pairs, min)
         if should_replace:
             self._remove_non_existent_tables_from_db()
+
         self.db_connection.commit()
 
         with tokenizer.name_analyzer() as analyzer:
@@ -163,7 +188,8 @@ class SPImporter():
         return (phrase.p_class, phrase.p_type)
 
     def _create_classtype_table_and_indexes(self,
-                                            class_type_pairs: Iterable[Tuple[str, str]]) -> None:
+                                            class_type_pairs: Iterable[Tuple[str, str]],
+                                            min: int = 0) -> None:
         """
             Create table place_classtype for each given pair.
             Also create indexes on place_id and centroid.
@@ -177,10 +203,19 @@ class SPImporter():
         with self.db_connection.cursor() as db_cursor:
             db_cursor.execute("CREATE INDEX idx_placex_classtype ON placex (class, type)")
 
+        if min:
+            allowed_special_phrases = self.get_classtype_pairs(min)
+
         for pair in class_type_pairs:
             phrase_class = pair[0]
             phrase_type = pair[1]
 
+            # Will only filter if min is not 0
+            if min and (phrase_class, phrase_type) not in allowed_special_phrases:
+                LOG.warning("Skipping phrase %s=%s: not in allowed special phrases",
+                            phrase_class, phrase_type)
+                continue
+
             table_name = _classtype_table(phrase_class, phrase_type)
 
             if table_name in self.table_phrases_to_delete:

src/nominatim_db/tools/tiger_data.py

@@ -108,8 +108,7 @@ async def add_tiger_data(data_dir: str, config: Configuration, threads: int,
 
         async with QueryPool(dsn, place_threads, autocommit=True) as pool:
             with tokenizer.name_analyzer() as analyzer:
-                lines = 0
-                for row in tar:
+                for lineno, row in enumerate(tar, 1):
                     try:
                         address = dict(street=row['street'], postcode=row['postcode'])
                         args = ('SRID=4326;' + row['geometry'],
@@ -124,10 +123,8 @@ async def add_tiger_data(data_dir: str, config: Configuration, threads: int,
                                                     %s::INT, %s::TEXT, %s::JSONB, %s::TEXT)""",
                         args)
 
-                    lines += 1
-                    if lines == 1000:
+                    if not lineno % 1000:
                         print('.', end='', flush=True)
-                    lines = 0
 
         print('', flush=True)
 

src/nominatim_db/utils/centroid.py

@@ -30,8 +30,8 @@ class PointsCentroid:
         if self.count == 0:
             raise ValueError("No points available for centroid.")
 
-        return (float(self.sum_x/self.count)/10000000,
-                float(self.sum_y/self.count)/10000000)
+        return (self.sum_x / self.count / 10_000_000,
+                self.sum_y / self.count / 10_000_000)
 
     def __len__(self) -> int:
         return self.count
@@ -40,8 +40,8 @@ class PointsCentroid:
         if isinstance(other, Collection) and len(other) == 2:
             if all(isinstance(p, (float, int)) for p in other):
                 x, y = other
-                self.sum_x += int(x * 10000000)
-                self.sum_y += int(y * 10000000)
+                self.sum_x += int(x * 10_000_000)
+                self.sum_y += int(y * 10_000_000)
                 self.count += 1
                 return self
 

src/nominatim_db/version.py

@@ -55,7 +55,7 @@ def parse_version(version: str) -> NominatimVersion:
     return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')])
 
 
-NOMINATIM_VERSION = parse_version('5.0.0-0')
+NOMINATIM_VERSION = parse_version('5.1.0-0')
 
 POSTGRESQL_REQUIRED_VERSION = (12, 0)
 POSTGIS_REQUIRED_VERSION = (3, 0)

test/Makefile

@@ -1,10 +0,0 @@
-all: bdd python
-
-bdd:
-	cd bdd && behave -DREMOVE_TEMPLATE=1
-
-python:
-	pytest python
-
-
-.PHONY: bdd python

test/bdd/.behaverc

@@ -1,3 +0,0 @@
-[behave]
-show_skipped=False
-default_tags=~@Fail

test/bdd/api/details/language.feature

@@ -1,63 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Localization of search results
-
-    Scenario: default language
-        When sending details query for R1155955
-        Then results contain
-          | ID | localname |
-          | 0  | Liechtenstein |
-
-    Scenario: accept-language first
-        When sending details query for R1155955
-          | accept-language |
-          | zh,de |
-        Then results contain
-          | ID | localname |
-          | 0  | 列支敦士登 |
-
-    Scenario: accept-language missing
-        When sending details query for R1155955
-          | accept-language |
-          | xx,fr,en,de |
-        Then results contain
-          | ID | localname |
-          | 0  | Liechtenstein |
-
-    Scenario: http accept language header first
-        Given the HTTP header
-          | accept-language |
-          | fo;q=0.8,en-ca;q=0.5,en;q=0.3 |
-        When sending details query for R1155955
-        Then results contain
-          | ID | localname |
-          | 0  | Liktinstein |
-
-    Scenario: http accept language header and accept-language
-        Given the HTTP header
-          | accept-language |
-          | fr-ca,fr;q=0.8,en-ca;q=0.5,en;q=0.3 |
-        When sending details query for R1155955
-          | accept-language |
-          | fo,en |
-        Then results contain
-          | ID | localname |
-          | 0  | Liktinstein |
-
-    Scenario: http accept language header fallback
-        Given the HTTP header
-          | accept-language |
-          | fo-ca,en-ca;q=0.5 |
-        When sending details query for R1155955
-        Then results contain
-          | ID | localname |
-          | 0  | Liktinstein |
-
-    Scenario: http accept language header fallback (upper case)
-        Given the HTTP header
-          | accept-language |
-          | fo-FR;q=0.8,en-ca;q=0.5 |
-        When sending details query for R1155955
-        Then results contain
-          | ID | localname |
-          | 0  | Liktinstein |

test/bdd/api/details/params.feature

@@ -1,96 +0,0 @@
-@APIDB
-Feature: Object details
-    Testing different parameter options for details API.
-
-    @SQLITE
-    Scenario: JSON Details
-        When sending json details query for W297699560
-        Then the result is valid json
-        And result has attributes geometry
-        And result has not attributes keywords,address,linked_places,parentof
-        And results contain in field geometry
-            | type  |
-            | Point |
-
-    @SQLITE
-    Scenario: JSON Details with pretty printing
-        When sending json details query for W297699560
-            | pretty |
-            | 1      |
-        Then the result is valid json
-        And result has attributes geometry
-        And result has not attributes keywords,address,linked_places,parentof
-
-    @SQLITE
-     Scenario: JSON Details with addressdetails
-        When sending json details query for W297699560
-            | addressdetails |
-            | 1 |
-        Then the result is valid json
-        And result has attributes address
-
-    @SQLITE
-    Scenario: JSON Details with linkedplaces
-        When sending json details query for R123924
-            | linkedplaces |
-            | 1 |
-        Then the result is valid json
-        And result has attributes linked_places
-
-    @SQLITE
-    Scenario: JSON Details with hierarchy
-        When sending json details query for W297699560
-            | hierarchy |
-            | 1 |
-        Then the result is valid json
-        And result has attributes hierarchy
-
-    @SQLITE
-    Scenario: JSON Details with grouped hierarchy
-        When sending json details query for W297699560
-            | hierarchy | group_hierarchy |
-            | 1         | 1 |
-        Then the result is valid json
-        And result has attributes hierarchy
-
-     Scenario Outline: JSON Details with keywords
-        When sending json details query for <osmid>
-            | keywords |
-            | 1 |
-        Then the result is valid json
-        And result has attributes keywords
-
-    Examples:
-            | osmid |
-            | W297699560 |
-            | W243055645 |
-            | W243055716 |
-            | W43327921  |
-
-    # ticket #1343
-    Scenario: Details of a country with keywords
-        When sending details query for R1155955
-            | keywords |
-            | 1 |
-        Then the result is valid json
-        And result has attributes keywords
-
-    @SQLITE
-    Scenario Outline: JSON details with full geometry
-        When sending json details query for <osmid>
-            | polygon_geojson |
-            | 1 |
-        Then the result is valid json
-        And result has attributes geometry
-        And results contain in field geometry
-            | type       |
-            | <geometry> |
-
-    Examples:
-            | osmid      | geometry   |
-            | W297699560 | LineString |
-            | W243055645 | Polygon    |
-            | W243055716 | Polygon    |
-            | W43327921  | LineString |
-
-

test/bdd/api/details/simple.feature

@@ -1,81 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Object details
-    Check details page for correctness
-
-    Scenario Outline: Details via OSM id
-        When sending details query for <type><id>
-        Then the result is valid json
-        And results contain
-            | osm_type | osm_id |
-            | <type>   | <id> |
-
-    Examples:
-     | type | id |
-     | N    | 5484325405 |
-     | W    | 43327921 |
-     | R    | 123924 |
-
-
-    Scenario Outline: Details for different class types for the same OSM id
-        When sending details query for N300209696:<class>
-        Then the result is valid json
-        And results contain
-          | osm_type | osm_id    | category |
-          | N        | 300209696 | <class> |
-
-    Examples:
-     | class |
-     | tourism |
-     | mountain_pass |
-
-
-    Scenario Outline: Details via unknown OSM id
-        When sending details query for <object>
-        Then a HTTP 404 is returned
-
-    Examples:
-      | object |
-      | 1 |
-      | R1 |
-      | N300209696:highway |
-
-
-     Scenario: Details for interpolation way return the interpolation
-        When sending details query for W1
-        Then the result is valid json
-        And results contain
-            | category | type   | osm_type | osm_id | admin_level |
-            | place    | houses | W        | 1      | 15          |
-
-
-     @Fail
-     Scenario: Details for interpolation way return the interpolation
-        When sending details query for 112871
-        Then the result is valid json
-        And results contain
-            | category | type   | admin_level |
-            | place    | houses | 15          |
-        And result has not attributes osm_type,osm_id
-
-
-     @Fail
-     Scenario: Details for interpolation way return the interpolation
-        When sending details query for 112820
-        Then the result is valid json
-        And results contain
-            | category | type     | admin_level |
-            | place    | postcode | 15          |
-        And result has not attributes osm_type,osm_id
-
-
-    Scenario Outline: Details debug output returns no errors
-        When sending debug details query for <feature>
-        Then the result is valid html
-
-        Examples:
-          | feature     |
-          | N5484325405 |
-          | W1          |
-          | 112820      |
-          | 112871      |

test/bdd/api/errors/formats.feature

@@ -1,14 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Places by osm_type and osm_id Tests
-    Simple tests for errors in various response formats.
-
-    Scenario Outline: Force error by providing too many ids
-        When sending <format> lookup query for N1,N2,N3,N4,N5,N6,N7,N8,N9,N10,N11,N12,N13,N14,N15,N16,N17,N18,N19,N20,N21,N22,N23,N24,N25,N26,N27,N28,N29,N30,N31,N32,N33,N34,N35,N36,N37,N38,N39,N40,N41,N42,N43,N44,N45,N46,N47,N48,N49,N50,N51
-        Then a <format> user error is returned
-
-    Examples:
-        | format  |
-        | xml     |
-        | json    |
-        | geojson |

test/bdd/api/lookup/simple.feature

@@ -1,42 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Places by osm_type and osm_id Tests
-    Simple tests for response format.
-
-    Scenario Outline: address lookup for existing node, way, relation
-        When sending <format> lookup query for N5484325405,W43327921,,R123924,X99,N0
-        Then the result is valid <outformat>
-        And exactly 3 results are returned
-
-    Examples:
-        | format      | outformat   |
-        | xml         | xml         |
-        | json        | json        |
-        | jsonv2      | json        |
-        | geojson     | geojson     |
-        | geocodejson | geocodejson |
-
-    Scenario: address lookup for non-existing or invalid node, way, relation
-        When sending xml lookup query for X99,,N0,nN158845944,ABC,,W9
-        Then exactly 0 results are returned
-
-    Scenario Outline: Boundingbox is returned
-        When sending <format> lookup query for N5484325405,W43327921
-        Then exactly 2 results are returned
-        And result 0 has bounding box in 47.135,47.14,9.52,9.525
-        And result 1 has bounding box in 47.07,47.08,9.50,9.52
-
-    Examples:
-      | format |
-      | json |
-      | jsonv2 |
-      | geojson |
-      | xml |
-
-
-    Scenario: Lookup of a linked place
-        When sending geocodejson lookup query for N1932181216
-        Then exactly 1 result is returned
-        And results contain
-          | name  |
-          | Vaduz |

test/bdd/api/reverse/geometry.feature

@@ -1,45 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Geometries for reverse geocoding
-    Tests for returning geometries with reverse
-
-
-    Scenario: Polygons are returned fully by default
-        When sending v1/reverse at 47.13803,9.52264
-          | polygon_text |
-          | 1            |
-        Then results contain
-          | geotext |
-          | ^POLYGON\(\(9.5225302 47.138066, ?9.5225348 47.1379282, ?9.5226142 47.1379294, ?9.5226143 47.1379257, ?9.522615 47.137917, ?9.5226225 47.1379098, ?9.5226334 47.1379052, ?9.5226461 47.1379037, ?9.5226588 47.1379056, ?9.5226693 47.1379107, ?9.5226762 47.1379181, ?9.5226762 47.1379268, ?9.5226761 47.1379308, ?9.5227366 47.1379317, ?9.5227352 47.1379753, ?9.5227608 47.1379757, ?9.5227595 47.1380148, ?9.5227355 47.1380145, ?9.5227337 47.1380692, ?9.5225302 47.138066\)\) |
-
-
-    Scenario: Polygons can be slightly simplified
-        When sending v1/reverse at 47.13803,9.52264
-          | polygon_text | polygon_threshold |
-          | 1            | 0.00001            |
-        Then results contain
-          | geotext |
-          | ^POLYGON\(\(9.5225302 47.138066, ?9.5225348 47.1379282, ?9.5226142 47.1379294, ?9.5226225 47.1379098, ?9.5226588 47.1379056, ?9.5226761 47.1379308, ?9.5227366 47.1379317, ?9.5227352 47.1379753, ?9.5227608 47.1379757, ?9.5227595 47.1380148, ?9.5227355 47.1380145, ?9.5227337 47.1380692, ?9.5225302 47.138066\)\) |
-
-
-    Scenario: Polygons can be much simplified
-        When sending v1/reverse at 47.13803,9.52264
-          | polygon_text | polygon_threshold |
-          | 1            | 0.9               |
-        Then results contain
-          | geotext |
-          | ^POLYGON\(\([0-9. ]+, ?[0-9. ]+, ?[0-9. ]+, ?[0-9. ]+(, ?[0-9. ]+)?\)\) |
-
-
-    Scenario: For polygons return the centroid as center point
-        When sending v1/reverse at 47.13836,9.52304
-        Then results contain
-          | centroid               |
-          | 9.52271080 47.13818045 |
-
-
-    Scenario: For streets return the closest point as center point
-        When sending v1/reverse at 47.13368,9.52942
-        Then results contain
-          | centroid    |
-          | 9.529431527 47.13368172 |

test/bdd/api/reverse/language.feature

@@ -1,37 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Localization of reverse search results
-
-    Scenario: default language
-        When sending v1/reverse at 47.14,9.55
-        Then result addresses contain
-          | ID | country |
-          | 0  | Liechtenstein |
-
-    Scenario: accept-language parameter
-        When sending v1/reverse at 47.14,9.55
-          | accept-language |
-          | ja,en |
-        Then result addresses contain
-          | ID | country |
-          | 0  | リヒテンシュタイン |
-
-    Scenario: HTTP accept language header
-        Given the HTTP header
-          | accept-language |
-          | fo-ca,fo;q=0.8,en-ca;q=0.5,en;q=0.3 |
-        When sending v1/reverse at 47.14,9.55
-        Then result addresses contain
-          | ID | country |
-          | 0  | Liktinstein |
-
-    Scenario: accept-language parameter and HTTP header
-        Given the HTTP header
-          | accept-language |
-          | fo-ca,fo;q=0.8,en-ca;q=0.5,en;q=0.3 |
-        When sending v1/reverse at 47.14,9.55
-          | accept-language |
-          | en |
-        Then result addresses contain
-          | ID | country |
-          | 0  | Liechtenstein |

test/bdd/api/reverse/queries.feature

@@ -1,117 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Reverse geocoding
-    Testing the reverse function
-
-    Scenario Outline: Simple reverse-geocoding with no results
-        When sending v1/reverse at <lat>,<lon>
-        Then exactly 0 results are returned
-
-    Examples:
-     | lat      | lon |
-     | 0.0      | 0.0 |
-     | 91.3     | 0.4    |
-     | -700     | 0.4    |
-     | 0.2      | 324.44 |
-     | 0.2      | -180.4 |
-
-
-    Scenario: Unknown countries fall back to default country grid
-        When sending v1/reverse at 45.174,-103.072
-        Then results contain
-          | category | type    | display_name |
-          | place    | country | United States |
-
-
-    @Tiger
-    Scenario: TIGER house number
-        When sending v1/reverse at 32.4752389363,-86.4810198619
-        Then results contain
-          | category | type |
-          | place    | house |
-        And result addresses contain
-          | house_number | road                | postcode | country_code |
-          | 707          | Upper Kingston Road | 36067    | us |
-
-    @Tiger
-    Scenario: No TIGER house number for zoom < 18
-        When sending v1/reverse at 32.4752389363,-86.4810198619
-          | zoom |
-          | 17 |
-        Then results contain
-          | osm_type | category |
-          | way      | highway  |
-        And result addresses contain
-          | road                | postcode | country_code |
-          | Upper Kingston Road | 36067    | us |
-
-    Scenario: Interpolated house number
-        When sending v1/reverse at 47.118533,9.57056562
-        Then results contain
-          | osm_type | category | type |
-          | way      | place    | house |
-        And result addresses contain
-          | house_number | road |
-          | 1019         | Grosssteg |
-
-    Scenario: Address with non-numerical house number
-        When sending v1/reverse at 47.107465,9.52838521614
-        Then result addresses contain
-          | house_number | road |
-          | 39A/B        | Dorfstrasse |
-
-
-    Scenario: Address with numerical house number
-        When sending v1/reverse at 47.168440329479594,9.511551699184338
-        Then result addresses contain
-          | house_number | road |
-          | 6            | Schmedgässle |
-
-    Scenario Outline: Zoom levels below 5 result in country
-        When sending v1/reverse at 47.16,9.51
-         | zoom |
-         | <zoom> |
-        Then results contain
-         | display_name |
-         | Liechtenstein |
-
-    Examples:
-         | zoom |
-         | 0    |
-         | 1    |
-         | 2    |
-         | 3    |
-         | 4    |
-
-    Scenario: When on a street, the closest interpolation is shown
-        When sending v1/reverse at 47.118457166193245,9.570678289621355
-         | zoom |
-         | 18 |
-        Then results contain
-         | display_name |
-         | 1021, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |
-
-    # github 2214
-    Scenario: Interpolations do not override house numbers when they are closer
-        When sending v1/reverse at 47.11778,9.57255
-         | zoom |
-         | 18 |
-        Then results contain
-         | display_name |
-         | 5, Grosssteg, Steg, Triesenberg, Oberland, 9497, Liechtenstein |
-
-    Scenario: Interpolations do not override house numbers when they are closer (2)
-        When sending v1/reverse at 47.11834,9.57167
-         | zoom |
-         | 18 |
-        Then results contain
-         | display_name |
-         | 3, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |
-
-    Scenario: When on a street with zoom 18, the closest housenumber is returned
-        When sending v1/reverse at 47.11755503977281,9.572722250405036
-         | zoom |
-         | 18 |
-        Then result addresses contain
-         | house_number |
-         | 7 |

test/bdd/api/reverse/v1_geocodejson.feature

@@ -1,107 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Geocodejson for Reverse API
-    Testing correctness of geocodejson output (API version v1).
-
-    Scenario Outline: Simple OSM result
-        When sending v1/reverse at 47.066,9.504 with format geocodejson
-          | addressdetails |
-          | <has_address>  |
-        Then result has attributes place_id, accuracy
-        And result has <attributes> country,postcode,county,city,district,street,housenumber, admin
-        Then results contain
-          | osm_type | osm_id     | osm_key | osm_value | type  |
-          | node     | 6522627624 | shop    | bakery    | house |
-        And results contain
-          | name                  | label |
-          | Dorfbäckerei Herrmann | Dorfbäckerei Herrmann, 29, Gnetsch, Mäls, Balzers, Oberland, 9496, Liechtenstein |
-        And results contain in field geojson
-          | type  | coordinates             |
-          | Point | [9.5036065, 47.0660892] |
-        And results contain in field __geocoding
-          | version | licence | attribution |
-          | 0.1.0   | ODbL    | ^Data © OpenStreetMap contributors, ODbL 1.0. https?://osm.org/copyright$ |
-
-        Examples:
-          | has_address | attributes     |
-          | 1           | attributes     |
-          | 0           | not attributes |
-
-
-    Scenario: City housenumber-level address with street
-        When sending v1/reverse at 47.1068011,9.52810091 with format geocodejson
-        Then results contain
-          | housenumber | street    | postcode | city    | country |
-          | 8           | Im Winkel | 9495     | Triesen | Liechtenstein |
-         And results contain in field admin
-          | level6   | level8  |
-          | Oberland | Triesen |
-
-
-    Scenario: Town street-level address with street
-        When sending v1/reverse at 47.066,9.504 with format geocodejson
-          | zoom |
-          | 16 |
-        Then results contain
-          | name    | city    | postcode | country |
-          | Gnetsch | Balzers | 9496     | Liechtenstein |
-
-
-    Scenario: Poi street-level address with footway
-        When sending v1/reverse at 47.06515,9.50083 with format geocodejson
-        Then results contain
-          | street  | city    | postcode | country |
-          | Burgweg | Balzers | 9496     | Liechtenstein |
-
-
-    Scenario: City address with suburb
-        When sending v1/reverse at 47.146861,9.511771 with format geocodejson
-        Then results contain
-          | housenumber | street   | district | city  | postcode | country |
-          | 5           | Lochgass | Ebenholz | Vaduz | 9490     | Liechtenstein |
-
-
-    @Tiger
-    Scenario: Tiger address
-        When sending v1/reverse at 32.4752389363,-86.4810198619 with format geocodejson
-        Then results contain
-         | osm_type | osm_id    | osm_key | osm_value | type  |
-         | way      | 396009653 | place   | house     | house |
-        And results contain
-         | housenumber | street              | city       | county         | postcode | country       |
-         | 707         | Upper Kingston Road | Prattville | Autauga County | 36067    | United States |
-
-
-    Scenario: Interpolation address
-        When sending v1/reverse at 47.118533,9.57056562 with format geocodejson
-        Then results contain
-          | osm_type | osm_id | osm_key | osm_value | type  |
-          | way      | 1      | place   | house     | house |
-        And results contain
-          | label |
-          | 1019, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |
-        And result has not attributes name
-
-
-    Scenario: Line geometry output is supported
-        When sending v1/reverse at 47.06597,9.50467 with format geocodejson
-          | param           | value |
-          | polygon_geojson | 1     |
-        Then results contain in field geojson
-          | type       |
-          | LineString |
-
-
-    Scenario Outline: Only geojson polygons are supported
-        When sending v1/reverse at 47.06597,9.50467 with format geocodejson
-          | param   | value |
-          | <param> | 1     |
-        Then results contain in field geojson
-          | type  |
-          | Point |
-
-        Examples:
-          | param |
-          | polygon_text |
-          | polygon_svg  |
-          | polygon_kml  |

test/bdd/api/reverse/v1_geojson.feature

@@ -1,73 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Geojson for Reverse API
-    Testing correctness of geojson output (API version v1).
-
-    Scenario Outline: Simple OSM result
-        When sending v1/reverse at 47.066,9.504 with format geojson
-          | addressdetails |
-          | <has_address>  |
-        Then result has attributes place_id, importance, __licence
-        And result has <attributes> address
-        And results contain
-          | osm_type | osm_id     | place_rank | category | type    | addresstype |
-          | node     | 6522627624 | 30         | shop     | bakery  | shop        |
-        And results contain
-          | name                  | display_name |
-          | Dorfbäckerei Herrmann | Dorfbäckerei Herrmann, 29, Gnetsch, Mäls, Balzers, Oberland, 9496, Liechtenstein |
-        And results contain
-          | boundingbox |
-          | [47.0660392, 47.0661392, 9.5035565, 9.5036565] |
-        And results contain in field geojson
-          | type  | coordinates |
-          | Point | [9.5036065, 47.0660892] |
-
-        Examples:
-          | has_address | attributes     |
-          | 1           | attributes     |
-          | 0           | not attributes |
-
-
-    @Tiger
-    Scenario: Tiger address
-        When sending v1/reverse at 32.4752389363,-86.4810198619 with format geojson
-        Then results contain
-         | osm_type | osm_id    | category | type  | addresstype  | place_rank |
-         | way      | 396009653 | place    | house | place        | 30         |
-
-
-    Scenario: Interpolation address
-        When sending v1/reverse at 47.118533,9.57056562 with format geojson
-        Then results contain
-          | osm_type | osm_id | place_rank | category | type    | addresstype |
-          | way      | 1      | 30         | place    | house   | place       |
-        And results contain
-          | boundingbox |
-          | ^\[47.118495\d*, 47.118595\d*, 9.570496\d*, 9.570596\d*\] |
-        And results contain
-          | display_name |
-          | 1019, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |
-
-
-    Scenario: Line geometry output is supported
-        When sending v1/reverse at 47.06597,9.50467 with format geojson
-          | param           | value |
-          | polygon_geojson | 1     |
-        Then results contain in field geojson
-          | type       |
-          | LineString |
-
-
-    Scenario Outline: Only geojson polygons are supported
-        When sending v1/reverse at 47.06597,9.50467 with format geojson
-          | param   | value |
-          | <param> | 1     |
-        Then results contain in field geojson
-          | type  |
-          | Point |
-
-        Examples:
-          | param |
-          | polygon_text |
-          | polygon_svg  |
-          | polygon_kml  |

test/bdd/api/reverse/v1_json.feature

@@ -1,130 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Json output for Reverse API
-    Testing correctness of json and jsonv2 output (API version v1).
-
-    Scenario Outline: OSM result with and without addresses
-        When sending v1/reverse at 47.066,9.504 with format json
-          | addressdetails |
-          | <has_address>  |
-        Then result has <attributes> address
-        When sending v1/reverse at 47.066,9.504 with format jsonv2
-          | addressdetails |
-          | <has_address>  |
-        Then result has <attributes> address
-
-        Examples:
-          | has_address | attributes     |
-          | 1           | attributes     |
-          | 0           | not attributes |
-
-    Scenario Outline: Simple OSM result
-        When sending v1/reverse at 47.066,9.504 with format <format>
-        Then result has attributes place_id
-        And results contain
-          | licence |
-          | ^Data © OpenStreetMap contributors, ODbL 1.0. https?://osm.org/copyright$ |
-        And results contain
-          | osm_type | osm_id     |
-          | node     | 6522627624 |
-        And results contain
-          | centroid             | boundingbox |
-          | 9.5036065 47.0660892 | ['47.0660392', '47.0661392', '9.5035565', '9.5036565'] |
-        And results contain
-          | display_name |
-          | Dorfbäckerei Herrmann, 29, Gnetsch, Mäls, Balzers, Oberland, 9496, Liechtenstein |
-        And result has not attributes namedetails,extratags
-
-        Examples:
-          | format |
-          | json   |
-          | jsonv2 |
-
-    Scenario: Extra attributes of jsonv2 result
-        When sending v1/reverse at 47.066,9.504 with format jsonv2
-        Then result has attributes importance
-        Then results contain
-          | category | type   | name                  | place_rank | addresstype |
-          | shop     | bakery | Dorfbäckerei Herrmann | 30         | shop        |
-
-
-    @Tiger
-    Scenario: Tiger address
-        When sending v1/reverse at 32.4752389363,-86.4810198619 with format jsonv2
-        Then results contain
-         | osm_type | osm_id    | category | type  | addresstype  |
-         | way      | 396009653 | place    | house | place        |
-
-
-    Scenario Outline: Interpolation address
-        When sending v1/reverse at 47.118533,9.57056562 with format <format>
-        Then results contain
-          | osm_type | osm_id |
-          | way      | 1      |
-        And results contain
-          | centroid                | boundingbox |
-          | 9.57054676 47.118545392 | ^\['47.118495\d*', '47.118595\d*', '9.570496\d*', '9.570596\d*'\] |
-        And results contain
-          | display_name |
-          | 1019, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |
-
-        Examples:
-          | format |
-          | json   |
-          | jsonv2 |
-
-
-    Scenario Outline: Output of geojson
-       When sending v1/reverse at 47.06597,9.50467 with format <format>
-          | param           | value |
-          | polygon_geojson | 1     |
-       Then results contain in field geojson
-          | type       | coordinates |
-          | LineString | [[9.5039353, 47.0657546], [9.5040437, 47.0657781], [9.5040808, 47.065787], [9.5054298, 47.0661407]] |
-
-       Examples:
-          | format |
-          | json   |
-          | jsonv2 |
-
-
-    Scenario Outline: Output of WKT
-       When sending v1/reverse at 47.06597,9.50467 with format <format>
-          | param        | value |
-          | polygon_text | 1     |
-       Then results contain
-          | geotext |
-          | ^LINESTRING\(9.5039353 47.0657546, ?9.5040437 47.0657781, ?9.5040808 47.065787, ?9.5054298 47.0661407\) |
-
-       Examples:
-          | format |
-          | json   |
-          | jsonv2 |
-
-
-    Scenario Outline: Output of SVG
-       When sending v1/reverse at 47.06597,9.50467 with format <format>
-          | param       | value |
-          | polygon_svg | 1     |
-       Then results contain
-          | svg |
-          | M 9.5039353 -47.0657546 L 9.5040437 -47.0657781 9.5040808 -47.065787 9.5054298 -47.0661407 |
-
-       Examples:
-          | format |
-          | json   |
-          | jsonv2 |
-
-
-    Scenario Outline: Output of KML
-       When sending v1/reverse at 47.06597,9.50467 with format <format>
-          | param       | value |
-          | polygon_kml | 1     |
-       Then results contain
-          | geokml |
-          | ^<LineString><coordinates>9.5039\d*,47.0657\d* 9.5040\d*,47.0657\d* 9.5040\d*,47.065\d* 9.5054\d*,47.0661\d*</coordinates></LineString> |
-
-       Examples:
-          | format |
-          | json   |
-          | jsonv2 |

test/bdd/api/reverse/v1_params.feature

@@ -1,206 +0,0 @@
-@SQLITE
-@APIDB
-Feature: v1/reverse Parameter Tests
-    Tests for parameter inputs for the v1 reverse endpoint.
-    This file contains mostly bad parameter input. Valid parameters
-    are tested in the format tests.
-
-    Scenario: Bad format
-        When sending v1/reverse at 47.14122383,9.52169581334 with format sdf
-        Then a HTTP 400 is returned
-
-    Scenario: Missing lon parameter
-        When sending v1/reverse at 52.52,
-        Then a HTTP 400 is returned
-
-
-    Scenario: Missing lat parameter
-        When sending v1/reverse at ,52.52
-        Then a HTTP 400 is returned
-
-
-    Scenario Outline: Bad format for lat or lon
-        When sending v1/reverse at ,
-          | lat   | lon   |
-          | <lat> | <lon> |
-        Then a HTTP 400 is returned
-
-        Examples:
-          | lat      | lon |
-          | 48.9660  | 8,4482 |
-          | 48,9660  | 8.4482 |
-          | 48,9660  | 8,4482 |
-          | 48.966.0 | 8.4482 |
-          | 48.966   | 8.448.2 |
-          | Nan      | 8.448  |
-          | 48.966   | Nan    |
-          | Inf      | 5.6    |
-          | 5.6      | -Inf   |
-          | <script></script> | 3.4 |
-          | 3.4 | <script></script> |
-          | -45.3    | ;      |
-          | gkjd     | 50     |
-
-
-    Scenario: Non-numerical zoom levels return an error
-        When sending v1/reverse at 47.14122383,9.52169581334
-          | zoom |
-          | adfe |
-        Then a HTTP 400 is returned
-
-
-    Scenario Outline: Truthy values for boolean parameters
-        When sending v1/reverse at 47.14122383,9.52169581334
-          | addressdetails |
-          | <value> |
-        Then exactly 1 result is returned
-        And result has attributes address
-
-        When sending v1/reverse at 47.14122383,9.52169581334
-          | extratags |
-          | <value> |
-        Then exactly 1 result is returned
-        And result has attributes extratags
-
-        When sending v1/reverse at 47.14122383,9.52169581334
-          | namedetails |
-          | <value> |
-        Then exactly 1 result is returned
-        And result has attributes namedetails
-
-        When sending v1/reverse at 47.14122383,9.52169581334
-          | polygon_geojson |
-          | <value> |
-        Then exactly 1 result is returned
-        And result has attributes geojson
-
-        When sending v1/reverse at 47.14122383,9.52169581334
-          | polygon_kml |
-          | <value> |
-        Then exactly 1 result is returned
-        And result has attributes geokml
-
-        When sending v1/reverse at 47.14122383,9.52169581334
-          | polygon_svg |
-          | <value> |
-        Then exactly 1 result is returned
-        And result has attributes svg
-
-        When sending v1/reverse at 47.14122383,9.52169581334
-          | polygon_text |
-          | <value> |
-        Then exactly 1 result is returned
-        And result has attributes geotext
-
-        Examples:
-          | value |
-          | yes   |
-          | no    |
-          | -1    |
-          | 100   |
-          | false |
-          | 00    |
-
-
-    Scenario: Only one geometry can be requested
-        When sending v1/reverse at 47.165989816710066,9.515774846076965
-          | polygon_text | polygon_svg |
-          | 1            | 1           |
-        Then a HTTP 400 is returned
-
-
-    Scenario Outline: Wrapping of legal jsonp requests
-        When sending v1/reverse at 67.3245,0.456 with format <format>
-          | json_callback |
-          | foo |
-        Then the result is valid <outformat>
-
-        Examples:
-          | format      | outformat   |
-          | json        | json        |
-          | jsonv2      | json        |
-          | geojson     | geojson     |
-          | geocodejson | geocodejson |
-
-
-    Scenario Outline: Illegal jsonp are not allowed
-        When sending v1/reverse at 47.165989816710066,9.515774846076965
-          | param        | value |
-          |json_callback | <data> |
-        Then a HTTP 400 is returned
-
-        Examples:
-          | data |
-          | 1asd |
-          | bar(foo) |
-          | XXX['bad'] |
-          | foo; evil |
-
-
-    Scenario Outline: Reverse debug mode produces valid HTML
-        When sending v1/reverse at , with format debug
-          | lat   | lon   |
-          | <lat> | <lon> |
-        Then the result is valid html
-
-        Examples:
-          | lat      | lon     |
-          | 0.0      | 0.0     |
-          | 47.06645 | 9.56601 |
-          | 47.14081 | 9.52267 |
-
-
-    Scenario Outline: Full address display for city housenumber-level address with street
-        When sending v1/reverse at 47.1068011,9.52810091 with format <format>
-        Then address of result 0 is
-          | type           | value     |
-          | house_number   | 8         |
-          | road           | Im Winkel |
-          | neighbourhood  | Oberdorf  |
-          | village        | Triesen   |
-          | ISO3166-2-lvl8 | LI-09     |
-          | county         | Oberland  |
-          | postcode       | 9495      |
-          | country        | Liechtenstein |
-          | country_code   | li        |
-
-        Examples:
-          | format  |
-          | json    |
-          | jsonv2  |
-          | geojson |
-          | xml     |
-
-
-    Scenario Outline: Results with name details
-        When sending v1/reverse at 47.14052,9.52202 with format <format>
-          | zoom | namedetails |
-          | 14   | 1           |
-        Then results contain in field namedetails
-          | name     |
-          | Ebenholz |
-
-        Examples:
-          | format  |
-          | json    |
-          | jsonv2  |
-          | xml     |
-          | geojson |
-
-
-    Scenario Outline: Results with extratags
-        When sending v1/reverse at 47.14052,9.52202 with format <format>
-          | zoom | extratags |
-          | 14   | 1         |
-        Then results contain in field extratags
-          | wikidata |
-          | Q4529531 |
-
-        Examples:
-          | format |
-          | json   |
-          | jsonv2 |
-          | xml    |
-          | geojson |
-
-

test/bdd/api/reverse/v1_xml.feature

@@ -1,88 +0,0 @@
-@SQLITE
-@APIDB
-Feature: XML output for Reverse API
-    Testing correctness of xml output (API version v1).
-
-    Scenario Outline: OSM result with and without addresses
-        When sending v1/reverse at 47.066,9.504 with format xml
-          | addressdetails |
-          | <has_address>  |
-        Then result has attributes place_id
-        Then result has <attributes> address
-        And results contain
-          | osm_type | osm_id     | place_rank | address_rank |
-          | node     | 6522627624 | 30         | 30           |
-        And results contain
-          | centroid             | boundingbox |
-          | 9.5036065 47.0660892 | 47.0660392,47.0661392,9.5035565,9.5036565 |
-        And results contain
-          | ref                   | display_name |
-          | Dorfbäckerei Herrmann | Dorfbäckerei Herrmann, 29, Gnetsch, Mäls, Balzers, Oberland, 9496, Liechtenstein |
-
-        Examples:
-          | has_address | attributes     |
-          | 1           | attributes     |
-          | 0           | not attributes |
-
-
-    @Tiger
-    Scenario: Tiger address
-        When sending v1/reverse at 32.4752389363,-86.4810198619 with format xml
-        Then results contain
-         | osm_type | osm_id    | place_rank  | address_rank |
-         | way      | 396009653 | 30          | 30           |
-        And results contain
-          | centroid                     | boundingbox |
-          | -86.4808553 32.4753580 | ^32.4753080\d*,32.4754080\d*,-86.4809053\d*,-86.4808053\d* |
-        And results contain
-          | display_name |
-          | 707, Upper Kingston Road, Upper Kingston, Prattville, Autauga County, 36067, United States |
-
-
-    Scenario: Interpolation address
-        When sending v1/reverse at 47.118533,9.57056562 with format xml
-        Then results contain
-          | osm_type | osm_id | place_rank | address_rank |
-          | way      | 1      | 30         | 30           |
-        And results contain
-          | centroid                | boundingbox |
-          | 9.57054676 47.118545392 | ^47.118495\d*,47.118595\d*,9.570496\d*,9.570596\d* |
-        And results contain
-          | display_name |
-          | 1019, Grosssteg, Sücka, Triesenberg, Oberland, 9497, Liechtenstein |
-
-
-    Scenario: Output of geojson
-       When sending v1/reverse at 47.06597,9.50467 with format xml
-          | param           | value |
-          | polygon_geojson | 1     |
-       Then results contain
-          | geojson |
-          | {"type":"LineString","coordinates":[[9.5039353,47.0657546],[9.5040437,47.0657781],[9.5040808,47.065787],[9.5054298,47.0661407]]}  |
-
-
-    Scenario: Output of WKT
-       When sending v1/reverse at 47.06597,9.50467 with format xml
-          | param        | value |
-          | polygon_text | 1     |
-       Then results contain
-          | geotext |
-          | ^LINESTRING\(9.5039353 47.0657546, ?9.5040437 47.0657781, ?9.5040808 47.065787, ?9.5054298 47.0661407\) |
-
-
-    Scenario: Output of SVG
-       When sending v1/reverse at 47.06597,9.50467 with format xml
-          | param       | value |
-          | polygon_svg | 1     |
-       Then results contain
-          | geosvg |
-          | M 9.5039353 -47.0657546 L 9.5040437 -47.0657781 9.5040808 -47.065787 9.5054298 -47.0661407 |
-
-
-    Scenario: Output of KML
-       When sending v1/reverse at 47.06597,9.50467 with format xml
-          | param       | value |
-          | polygon_kml | 1     |
-       Then results contain
-          | geokml |
-          | ^<geokml><LineString><coordinates>9.5039\d*,47.0657\d* 9.5040\d*,47.0657\d* 9.5040\d*,47.065\d* 9.5054\d*,47.0661\d*</coordinates></LineString></geokml> |

test/bdd/api/search/geocodejson.feature

@@ -1,28 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Parameters for Search API
-    Testing correctness of geocodejson output.
-
-    Scenario: City housenumber-level address with street
-        When sending geocodejson search query "Im Winkel 8, Triesen" with address
-        Then results contain
-          | housenumber | street    | postcode | city    | country |
-          | 8           | Im Winkel | 9495     | Triesen | Liechtenstein |
-
-    Scenario: Town street-level address with street
-        When sending geocodejson search query "Gnetsch, Balzers" with address
-        Then results contain
-          | name    | city    | postcode | country |
-          | Gnetsch | Balzers | 9496     | Liechtenstein |
-
-    Scenario: Town street-level address with footway
-        When sending geocodejson search query "burg gutenberg 6000 jahre geschichte" with address
-        Then results contain
-          | street  | city    | postcode | country |
-          | Burgweg | Balzers | 9496     | Liechtenstein |
-
-    Scenario: City address with suburb
-        When sending geocodejson search query "Lochgass 5, Ebenholz, Vaduz" with address
-        Then results contain
-          | housenumber | street   | district | city  | postcode | country |
-          | 5           | Lochgass | Ebenholz | Vaduz | 9490     | Liechtenstein |

test/bdd/api/search/language.feature

@@ -1,63 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Localization of search results
-
-    Scenario: default language
-        When sending json search query "Liechtenstein"
-        Then results contain
-          | ID | display_name |
-          | 0  | Liechtenstein |
-
-    Scenario: accept-language first
-        When sending json search query "Liechtenstein"
-          | accept-language |
-          | zh,de |
-        Then results contain
-          | ID | display_name |
-          | 0  | 列支敦士登 |
-
-    Scenario: accept-language missing
-        When sending json search query "Liechtenstein"
-          | accept-language |
-          | xx,fr,en,de |
-        Then results contain
-          | ID | display_name |
-          | 0  | Liechtenstein |
-
-    Scenario: http accept language header first
-        Given the HTTP header
-          | accept-language |
-          | fo;q=0.8,en-ca;q=0.5,en;q=0.3 |
-        When sending json search query "Liechtenstein"
-        Then results contain
-          | ID | display_name |
-          | 0  | Liktinstein |
-
-    Scenario: http accept language header and accept-language
-        Given the HTTP header
-          | accept-language |
-          | fr-ca,fr;q=0.8,en-ca;q=0.5,en;q=0.3 |
-        When sending json search query "Liechtenstein"
-          | accept-language |
-          | fo,en |
-        Then results contain
-          | ID | display_name |
-          | 0  | Liktinstein |
-
-    Scenario: http accept language header fallback
-        Given the HTTP header
-          | accept-language |
-          | fo-ca,en-ca;q=0.5 |
-        When sending json search query "Liechtenstein"
-        Then results contain
-          | ID | display_name |
-          | 0  | Liktinstein |
-
-    Scenario: http accept language header fallback (upper case)
-        Given the HTTP header
-          | accept-language |
-          | fo-FR;q=0.8,en-ca;q=0.5 |
-        When sending json search query "Liechtenstein"
-        Then results contain
-          | ID | display_name |
-          | 0  | Liktinstein |

test/bdd/api/search/params.feature

@@ -1,362 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Search queries
-    Testing different queries and parameters
-
-    Scenario: Simple XML search
-        When sending xml search query "Schaan"
-        Then result 0 has attributes place_id,osm_type,osm_id
-        And result 0 has attributes place_rank,boundingbox
-        And result 0 has attributes lat,lon,display_name
-        And result 0 has attributes class,type,importance
-        And result 0 has not attributes address
-        And result 0 has bounding box in 46.5,47.5,9,10
-
-    Scenario: Simple JSON search
-        When sending json search query "Vaduz"
-        Then result 0 has attributes place_id,licence,class,type
-        And result 0 has attributes osm_type,osm_id,boundingbox
-        And result 0 has attributes lat,lon,display_name,importance
-        And result 0 has not attributes address
-        And result 0 has bounding box in 46.5,47.5,9,10
-
-    Scenario: Unknown formats returns a user error
-        When sending search query "Vaduz"
-          | format |
-          | x45    |
-        Then a HTTP 400 is returned
-
-    Scenario Outline: Search with addressdetails
-        When sending <format> search query "Triesen" with address
-        Then address of result 0 is
-          | type         | value |
-          | village      | Triesen |
-          | county       | Oberland |
-          | postcode     | 9495 |
-          | country      | Liechtenstein |
-          | country_code | li |
-          | ISO3166-2-lvl8 | LI-09 |
-
-    Examples:
-          | format |
-          | json   |
-          | jsonv2 |
-          | geojson |
-          | xml |
-
-    Scenario: Coordinate search with addressdetails
-        When sending json search query "47.12400621,9.6047552"
-          | accept-language |
-          | en |
-        Then results contain
-          | display_name |
-          | Guschg, Valorschstrasse, Balzers, Oberland, 9497, Liechtenstein |
-
-    Scenario: Address details with unknown class types
-        When sending json search query "Kloster St. Elisabeth" with address
-        Then results contain
-          | ID | class   | type |
-          | 0  | amenity | monastery |
-        And result addresses contain
-          | ID | amenity |
-          | 0  | Kloster St. Elisabeth |
-
-    Scenario: Disabling deduplication
-        When sending json search query "Malbunstr"
-        Then there are no duplicates
-        When sending json search query "Malbunstr"
-          | dedupe |
-          | 0 |
-        Then there are duplicates
-
-    Scenario: Search with bounded viewbox in right area
-        When sending json search query "post" with address
-          | bounded | viewbox |
-          | 1       |  9,47,10,48 |
-        Then result addresses contain
-          | ID | town |
-          | 0  | Vaduz |
-        When sending json search query "post" with address
-          | bounded | viewbox |
-          | 1       |  9.49712,47.17122,9.52605,47.16242 |
-        Then result addresses contain
-          | town |
-          | Schaan |
-
-    Scenario: Country search with bounded viewbox remain in the area
-        When sending json search query "" with address
-          | bounded | viewbox                                 | country |
-          | 1       | 9.49712,47.17122,9.52605,47.16242 | de |
-        Then less than 1 result is returned
-
-    Scenario: Search with bounded viewboxlbrt in right area
-        When sending json search query "bar" with address
-          | bounded | viewboxlbrt |
-          | 1       | 9.49712,47.16242,9.52605,47.17122 |
-        Then result addresses contain
-          | town |
-          | Schaan |
-
-    @Fail
-    Scenario: No POI search with unbounded viewbox
-        When sending json search query "restaurant"
-          | viewbox |
-          | 9.93027,53.61634,10.10073,53.54500 |
-        Then results contain
-          | display_name |
-          | ^[^,]*[Rr]estaurant.* |
-
-    Scenario: bounded search remains within viewbox, even with no results
-         When sending json search query "[restaurant]"
-           | bounded | viewbox |
-           | 1       | 43.5403125,-5.6563282,43.54285,-5.662003 |
-        Then less than 1 result is returned
-
-    Scenario: bounded search remains within viewbox with results
-        When sending json search query "restaurant"
-         | bounded | viewbox |
-         | 1       | 9.49712,47.17122,9.52605,47.16242 |
-        Then result has centroid in 9.49712,47.16242,9.52605,47.17122
-
-    Scenario: Prefer results within viewbox
-        When sending json search query "Gässle" with address
-          | accept-language | viewbox |
-          | en              | 9.52413,47.10759,9.53140,47.10539 |
-        Then result addresses contain
-          | ID | village |
-          | 0  | Triesen |
-        When sending json search query "Gässle" with address
-          | accept-language | viewbox |
-          | en              | 9.45949,47.08421,9.54094,47.05466 |
-        Then result addresses contain
-          | ID | town |
-          | 0  | Balzers |
-
-    Scenario: viewboxes cannot be points
-        When sending json search query "foo"
-          | viewbox |
-          | 1.01,34.6,1.01,34.6 |
-        Then a HTTP 400 is returned
-
-    Scenario Outline: viewbox must have four coordinate numbers
-        When sending json search query "foo"
-          | viewbox |
-          | <viewbox> |
-        Then a HTTP 400 is returned
-
-    Examples:
-        | viewbox |
-        | 34      |
-        | 0.003,-84.4 |
-        | 5.2,4.5542,12.4 |
-        | 23.1,-6,0.11,44.2,9.1 |
-
-    Scenario Outline: viewboxlbrt must have four coordinate numbers
-        When sending json search query "foo"
-          | viewboxlbrt |
-          | <viewbox> |
-        Then a HTTP 400 is returned
-
-    Examples:
-        | viewbox |
-        | 34      |
-        | 0.003,-84.4 |
-        | 5.2,4.5542,12.4 |
-        | 23.1,-6,0.11,44.2,9.1 |
-
-    Scenario: Overly large limit number for search results
-        When sending json search query "restaurant"
-          | limit |
-          | 1000 |
-        Then at most 50 results are returned
-
-    Scenario: Limit number of search results
-        When sending json search query "landstr"
-          | dedupe |
-          | 0      |
-        Then more than 4 results are returned
-        When sending json search query "landstr"
-          | limit | dedupe |
-          | 4     | 0      |
-        Then exactly 4 results are returned
-
-    Scenario: Limit parameter must be a number
-        When sending search query "Blue Laguna"
-          | limit |
-          | );    |
-        Then a HTTP 400 is returned
-
-    Scenario: Restrict to feature type country
-        When sending xml search query "fürstentum"
-          | featureType |
-          | country |
-        Then results contain
-          | place_rank |
-          | 4 |
-
-    Scenario: Restrict to feature type state
-        When sending xml search query "Wangerberg"
-        Then at least 1 result is returned
-        When sending xml search query "Wangerberg"
-          | featureType |
-          | state |
-        Then exactly 0 results are returned
-
-    Scenario: Restrict to feature type city
-        When sending xml search query "vaduz"
-        Then at least 1 result is returned
-        When sending xml search query "vaduz"
-          | featureType |
-          | city |
-        Then results contain
-          | place_rank |
-          | 16 |
-
-    Scenario: Restrict to feature type settlement
-        When sending json search query "Malbun"
-        Then results contain
-          | ID | class |
-          | 1  | landuse |
-        When sending json search query "Malbun"
-          | featureType |
-          | settlement |
-        Then results contain
-          | class | type |
-          | place | village |
-
-    Scenario Outline: Search with polygon threshold (json)
-        When sending json search query "triesenberg"
-          | polygon_geojson | polygon_threshold |
-          | 1               | <th> |
-        Then at least 1 result is returned
-        And result 0 has attributes geojson
-
-     Examples:
-        | th |
-        | -1 |
-        | 0.0 |
-        | 0.5 |
-        | 999 |
-
-    Scenario Outline: Search with polygon threshold (xml)
-        When sending xml search query "triesenberg"
-          | polygon_geojson | polygon_threshold |
-          | 1               | <th> |
-        Then at least 1 result is returned
-        And result 0 has attributes geojson
-
-     Examples:
-        | th |
-        | -1 |
-        | 0.0 |
-        | 0.5 |
-        | 999 |
-
-    Scenario Outline: Search with invalid polygon threshold (xml)
-        When sending xml search query "triesenberg"
-          | polygon_geojson | polygon_threshold |
-          | 1               | <th> |
-        Then a HTTP 400 is returned
-
-     Examples:
-        | th |
-        | x |
-        | ;; |
-        | 1m |
-
-    Scenario Outline: Search with extratags
-        When sending <format> search query "Landstr"
-          | extratags |
-          | 1 |
-        Then result has attributes extratags
-
-    Examples:
-        | format |
-        | xml |
-        | json |
-        | jsonv2 |
-        | geojson |
-
-    Scenario Outline: Search with namedetails
-        When sending <format> search query "Landstr"
-          | namedetails |
-          | 1 |
-        Then result has attributes namedetails
-
-    Examples:
-        | format |
-        | xml |
-        | json |
-        | jsonv2 |
-        | geojson |
-
-    Scenario Outline: Search result with contains TEXT geometry
-        When sending <format> search query "triesenberg"
-          | polygon_text |
-          | 1 |
-        Then result has attributes <response_attribute>
-
-    Examples:
-        | format   | response_attribute |
-        | xml      | geotext |
-        | json     | geotext |
-        | jsonv2   | geotext |
-
-    Scenario Outline: Search result contains SVG geometry
-        When sending <format> search query "triesenberg"
-          | polygon_svg |
-          | 1 |
-        Then result has attributes <response_attribute>
-
-    Examples:
-        | format   | response_attribute |
-        | xml      | geosvg |
-        | json     | svg |
-        | jsonv2   | svg |
-
-    Scenario Outline: Search result contains KML geometry
-        When sending <format> search query "triesenberg"
-          | polygon_kml |
-          | 1 |
-        Then result has attributes <response_attribute>
-
-    Examples:
-        | format   | response_attribute |
-        | xml      | geokml |
-        | json     | geokml |
-        | jsonv2   | geokml |
-
-    Scenario Outline: Search result contains GEOJSON geometry
-        When sending <format> search query "triesenberg"
-          | polygon_geojson |
-          | 1 |
-        Then result has attributes <response_attribute>
-
-    Examples:
-        | format   | response_attribute |
-        | xml      | geojson |
-        | json     | geojson |
-        | jsonv2   | geojson |
-        | geojson  | geojson |
-
-    Scenario Outline: Search result in geojson format contains no non-geojson geometry
-        When sending geojson search query "triesenberg"
-          | polygon_text | polygon_svg | polygon_geokml |
-          | 1            | 1           | 1              |
-        Then result 0 has not attributes <response_attribute>
-
-    Examples:
-        | response_attribute |
-        | geotext            |
-        | polygonpoints      |
-        | svg                |
-        | geokml             |
-
-
-    Scenario: Array parameters are ignored
-        When sending json search query "Vaduz" with address
-          | countrycodes[] | polygon_svg[] | limit[] | polygon_threshold[] |
-          | IT             | 1             | 3       | 3.4                 |
-        Then result addresses contain
-          | ID | country_code |
-          | 0  | li           |

test/bdd/api/search/queries.feature

@@ -1,221 +0,0 @@
-@SQLITE
-@APIDB
-Feature: Search queries
-    Generic search result correctness
-
-    Scenario: Search for natural object
-        When sending json search query "Samina"
-          | accept-language |
-          | en |
-        Then results contain
-          | ID | class    | type  | display_name    |
-          | 0  | waterway | river | Samina, Austria |
-
-    Scenario: House number search for non-street address
-        When sending json search query "6 Silum, Liechtenstein" with address
-          | accept-language |
-          | en |
-        Then address of result 0 is
-          | type         | value |
-          | house_number | 6 |
-          | village      | Silum |
-          | town         | Triesenberg |
-          | county       | Oberland |
-          | postcode     | 9497 |
-          | country      | Liechtenstein |
-          | country_code | li |
-          | ISO3166-2-lvl8  | LI-10 |
-
-    Scenario: House number interpolation
-        When sending json search query "Grosssteg 1023, Triesenberg" with address
-          | accept-language |
-          | de |
-        Then address of result 0 contains
-          | type          | value |
-          | house_number  | 1023 |
-          | road          | Grosssteg |
-          | village       | Sücka |
-          | postcode      | 9497 |
-          | town          | Triesenberg |
-          | country       | Liechtenstein |
-          | country_code  | li |
-
-    Scenario: With missing housenumber search falls back to road
-        When sending json search query "Bündaweg 555" with address
-        Then address of result 0 is
-          | type          | value |
-          | road          | Bündaweg |
-          | village       | Silum |
-          | postcode      | 9497 |
-          | county        | Oberland |
-          | town          | Triesenberg |
-          | country       | Liechtenstein |
-          | country_code  | li |
-          | ISO3166-2-lvl8  | LI-10 |
-
-    Scenario Outline: Housenumber 0 can be found
-        When sending <format> search query "Gnalpstrasse 0" with address
-        Then results contain
-          | display_name |
-          | ^0,.* |
-        And result addresses contain
-          | house_number |
-          | 0     |
-
-    Examples:
-        | format |
-        | xml |
-        | json |
-        | jsonv2 |
-        | geojson |
-
-    @Tiger
-    Scenario: TIGER house number
-        When sending json search query "697 Upper Kingston Road"
-        Then results contain
-         | osm_type | display_name |
-         | way      | ^697,.* |
-
-    Scenario: Search with class-type feature
-        When sending jsonv2 search query "bars in ebenholz"
-        Then results contain
-          | place_rank |
-          | 30 |
-
-    Scenario: Search with specific amenity
-        When sending json search query "[restaurant] Vaduz" with address
-        Then result addresses contain
-          | country |
-          | Liechtenstein |
-        And  results contain
-          | class   | type |
-          | amenity | restaurant |
-
-    Scenario: Search with specific amenity also work in country
-        When sending json search query "restaurants in liechtenstein" with address
-        Then result addresses contain
-          | country |
-          | Liechtenstein |
-        And  results contain
-          | class   | type |
-          | amenity | restaurant |
-
-    Scenario: Search with key-value amenity
-        When sending json search query "[club=scout] Vaduz"
-        Then results contain
-          | class | type |
-          | club  | scout |
-
-    Scenario: POI search near given coordinate
-        When sending json search query "restaurant near 47.16712,9.51100"
-        Then results contain
-          | class   | type |
-          | amenity | restaurant |
-
-    Scenario: Arbitrary key/value search near given coordinate
-        When sending json search query "[leisure=firepit]   47.150° N 9.5340493° E"
-        Then results contain
-          | class   | type |
-          | leisure | firepit |
-
-
-    Scenario: POI search in a bounded viewbox
-        When sending json search query "restaurants"
-          | viewbox                           | bounded |
-          | 9.50830,47.15253,9.52043,47.14866 | 1 |
-        Then results contain
-          | class   | type       |
-          | amenity | restaurant |
-
-    Scenario Outline: Key/value search near given coordinate can be restricted to country
-        When sending json search query "[natural=peak] 47.06512,9.53965" with address
-          | countrycodes |
-          | <cc> |
-        Then result addresses contain
-          | country_code |
-          | <cc> |
-
-    Examples:
-        | cc |
-        | li |
-        | ch |
-
-    Scenario: Name search near given coordinate
-        When sending json search query "sporry" with address
-        Then result addresses contain
-          | ID | town |
-          | 0  | Vaduz |
-        When sending json search query "sporry, 47.10791,9.52676" with address
-        Then result addresses contain
-          | ID | village |
-          | 0  | Triesen |
-
-    Scenario: Name search near given coordinate without result
-        When sending json search query "sporry, N 47 15 7 W 9 61 26"
-        Then exactly 0 results are returned
-
-    Scenario: Arbitrary key/value search near a road
-        When sending json search query "[amenity=drinking_water] Wissfläckaweg"
-        Then results contain
-          | class   | type |
-          | amenity | drinking_water |
-
-    Scenario: Ignore other country codes in structured search with country
-        When sending json search query ""
-            | city | country |
-            | li   | de      |
-        Then exactly 0 results are returned
-
-    Scenario: Ignore country searches when query is restricted to countries
-        When sending json search query "fr"
-            | countrycodes |
-            | li  |
-        Then exactly 0 results are returned
-
-    Scenario: Country searches only return results for the given country
-        When sending search query "Ans Trail" with address
-            | countrycodes |
-            | li |
-        Then result addresses contain
-            | country_code |
-            | li |
-
-    # https://trac.openstreetmap.org/ticket/5094
-    Scenario: housenumbers are ordered by complete match first
-        When sending json search query "Austrasse 11, Vaduz" with address
-        Then result addresses contain
-          | ID | house_number |
-          | 0  | 11 |
-
-    Scenario Outline: Coordinate searches with white spaces
-        When sending json search query "<data>"
-        Then exactly 1 result is returned
-        And results contain
-          | class   |
-          | water |
-
-    Examples:
-      | data |
-      | sporry weiher, N 47.10791° E 9.52676° |
-      | sporry weiher,	N 47.10791° E 9.52676° |
-      | 	sporry weiher	, 	N 47.10791° E 9.52676° |
-      | sporry weiher, N 47.10791° 		E 9.52676° |
-      | sporry weiher, N 47.10791° E	9.52676° |
-
-    Scenario: Searches with white spaces
-        When sending json search query "52	Bodastr,Triesenberg"
-        Then results contain
-          | class   | type |
-          | highway | residential |
-
-
-    # github #1949
-    Scenario: Addressdetails always return the place type
-       When sending json search query "Vaduz" with address
-       Then result addresses contain
-         | ID | town |
-         | 0  | Vaduz |
-
-    Scenario: Search can handle complex query word sets
-       When sending search query "aussenstelle universitat lichtenstein wachterhaus aussenstelle universitat lichtenstein wachterhaus aussenstelle universitat lichtenstein wachterhaus aussenstelle universitat lichtenstein wachterhaus"
-       Then a HTTP 200 is returned