prepare release 4.2.4

Before postgis 3.4 ST_Project required a geography as input and seemed
to have implicitly converted to geography. Since 3.4 geometry input
is supported but leads to a completely different result.

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: lonvia
+custom: "https://nominatim.org/funding/"

.github/actions/build-nominatim/action.yml

@@ -5,34 +5,43 @@ inputs:
         description: 'Version of Ubuntu to install on'
         required: false
         default: '20'
+    cmake-args:
+        description: 'Additional options to hand to cmake'
+        required: false
+        default: ''
+    lua:
+        description: 'Version of Lua to use'
+        required: false
+        default: '5.3'
 
 runs:
     using: "composite"
 
     steps:
+        - name: Clean out the disk
+          run: |
+            sudo rm -rf /opt/hostedtoolcache/go /opt/hostedtoolcache/CodeQL /usr/lib/jvm /usr/local/share/chromium /usr/local/lib/android
+            df -h
+          shell: bash
         - name: Install prerequisites
           run: |
-            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev
+            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION}
             if [ "x$UBUNTUVER" == "x18" ]; then
-                pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium PyYAML==5.1 datrie
+                pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 datrie
             else
                 sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
             fi
           shell: bash
           env:
             UBUNTUVER: ${{ inputs.ubuntu }}
-
-        - name: Download dependencies
-          run: |
-              if [ ! -f country_grid.sql.gz ]; then
-                  wget --no-verbose https://www.nominatim.org/data/country_grid.sql.gz
-              fi
-              cp country_grid.sql.gz Nominatim/data/country_osm_grid.sql.gz
-          shell: bash
+            CMAKE_ARGS: ${{ inputs.cmake-args }}
+            LUA_VERSION: ${{ inputs.lua }}
 
         - name: Configure
-          run: mkdir build && cd build && cmake ../Nominatim
+          run: mkdir build && cd build && cmake $CMAKE_ARGS ../Nominatim
           shell: bash
+          env:
+            CMAKE_ARGS: ${{ inputs.cmake-args }}
 
         - name: Build
           run: |

.github/actions/setup-postgresql/action.yml

@@ -15,14 +15,16 @@ runs:
         - name: Remove existing PostgreSQL
           run: |
               sudo apt-get purge -yq postgresql*
-              sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+              sudo apt install curl ca-certificates gnupg
+              curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
+              sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
               sudo apt-get update -qq
 
           shell: bash
 
         - name: Install PostgreSQL
           run: |
-              sudo apt-get install -y -qq --no-install-suggests --no-install-recommends postgresql-client-${PGVER} postgresql-${PGVER}-postgis-${POSTGISVER} postgresql-${PGVER}-postgis-${POSTGISVER}-scripts postgresql-contrib-${PGVER} postgresql-${PGVER} postgresql-server-dev-${PGVER}
+              sudo apt-get install -y -qq --no-install-suggests --no-install-recommends postgresql-client-${PGVER} postgresql-${PGVER}-postgis-${POSTGISVER} postgresql-${PGVER}-postgis-${POSTGISVER}-scripts postgresql-contrib-${PGVER} postgresql-${PGVER}
           shell: bash
           env:
               PGVER: ${{ inputs.postgresql-version }}

.github/workflows/ci-tests.yml

@@ -7,11 +7,11 @@ jobs:
         runs-on: ubuntu-latest
 
         steps:
-            - uses: actions/checkout@v2
+            - uses: actions/checkout@v3
               with:
                 submodules: true
 
-            - uses: actions/cache@v2
+            - uses: actions/cache@v3
               with:
                   path: |
                      data/country_osm_grid.sql.gz
@@ -27,7 +27,7 @@ jobs:
                   mv nominatim-src.tar.bz2 Nominatim
 
             - name: 'Upload Artifact'
-              uses: actions/upload-artifact@v2
+              uses: actions/upload-artifact@v3
               with:
                   name: full-source
                   path: nominatim-src.tar.bz2
@@ -37,23 +37,23 @@ jobs:
         needs: create-archive
         strategy:
             matrix:
-                ubuntu: [18, 20]
+                ubuntu: [20, 22]
                 include:
-                    - ubuntu: 18
-                      postgresql: 9.5
-                      postgis: 2.5
-                      pytest: pytest
-                      php: 7.2
                     - ubuntu: 20
                       postgresql: 13
                       postgis: 3
                       pytest: py.test-3
                       php: 7.4
+                    - ubuntu: 22
+                      postgresql: 15
+                      postgis: 3
+                      pytest: py.test-3
+                      php: 8.1
 
         runs-on: ubuntu-${{ matrix.ubuntu }}.04
 
         steps:
-            - uses: actions/download-artifact@v2
+            - uses: actions/download-artifact@v3
               with:
                   name: full-source
 
@@ -64,10 +64,12 @@ jobs:
               uses: shivammathur/setup-php@v2
               with:
                   php-version: ${{ matrix.php }}
-                  coverage: xdebug
-                  tools: phpunit, phpcs, composer
+                  tools: phpunit:9, phpcs, composer
+                  ini-values: opcache.jit=disable
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-            - uses: actions/setup-python@v2
+            - uses: actions/setup-python@v4
               with:
                 python-version: 3.6
               if: matrix.ubuntu == 18
@@ -82,75 +84,60 @@ jobs:
                   ubuntu: ${{ matrix.ubuntu }}
 
             - name: Install test prerequsites
-              run: sudo apt-get install -y -qq pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage
+              run: sudo apt-get install -y -qq python3-pytest python3-behave
               if: matrix.ubuntu == 20
 
             - name: Install test prerequsites
-              run: pip3 install pylint==2.6.0 pytest pytest-cov behave==1.2.6
-              if: matrix.ubuntu == 18
+              run: pip3 install pylint pytest behave==1.2.6
+              if: ${{ (matrix.ubuntu == 18) || (matrix.ubuntu == 22) }}
+
+            - name: Install test prerequsites
+              run: sudo apt-get install -y -qq python3-pytest
+              if: matrix.ubuntu == 22
+
+            - name: Install latest pylint/mypy
+              run: pip3 install -U pylint mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil types-requests typing-extensions
+              if: matrix.ubuntu == 22
 
             - name: PHP linting
               run: phpcs --report-width=120 .
               working-directory: Nominatim
+              if: matrix.ubuntu == 22
 
             - name: Python linting
               run: pylint nominatim
               working-directory: Nominatim
+              if: matrix.ubuntu == 22
+
+            - name: Python static typechecking
+              run: mypy --strict nominatim
+              working-directory: Nominatim
+              if: matrix.ubuntu == 22
+
 
             - name: PHP unit tests
-              run: phpunit --coverage-clover ../../coverage-php.xml ./
+              run: phpunit ./
               working-directory: Nominatim/test/php
-              if: matrix.ubuntu == 20
+              if: ${{ (matrix.ubuntu == 20) || (matrix.ubuntu == 22) }}
 
             - name: Python unit tests
-              run: $PYTEST --cov=nominatim --cov-report=xml test/python
+              run: $PYTEST test/python
               working-directory: Nominatim
               env:
                 PYTEST: ${{ matrix.pytest }}
 
-            - name: BDD tests
-              run: |
-                  mkdir cov
-                  behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 -DPHPCOV=./cov
-                  composer require phpunit/phpcov:7.0.2
-                  vendor/bin/phpcov merge --clover ../../coverage-bdd.xml ./cov
-              working-directory: Nominatim/test/bdd
-              if: matrix.ubuntu == 20
-
             - name: BDD tests
               run: |
                   behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
               working-directory: Nominatim/test/bdd
-              if: matrix.ubuntu == 18
-
-            - name: Upload coverage to Codecov
-              uses: codecov/codecov-action@v1
-              with:
-                files: ./Nominatim/coverage*.xml
-                directory: ./
-                name: codecov-umbrella
-                fail_ci_if_error: false
-                path_to_write_report: ./coverage/codecov_report.txt
-                verbose: true
-              if: matrix.ubuntu == 20
 
 
-    icu-test:
+    legacy-test:
         needs: create-archive
-        strategy:
-            matrix:
-                ubuntu: [20]
-                include:
-                    - ubuntu: 20
-                      postgresql: 13
-                      postgis: 3
-                      pytest: py.test-3
-                      php: 7.4
-
-        runs-on: ubuntu-${{ matrix.ubuntu }}.04
+        runs-on: ubuntu-20.04
 
         steps:
-            - uses: actions/download-artifact@v2
+            - uses: actions/download-artifact@v3
               with:
                   name: full-source
 
@@ -160,35 +147,27 @@ jobs:
             - name: Setup PHP
               uses: shivammathur/setup-php@v2
               with:
-                  php-version: ${{ matrix.php }}
-                  coverage: xdebug
-                  tools: phpunit, phpcs, composer
-
-            - uses: actions/setup-python@v2
-              with:
-                python-version: 3.6
-              if: matrix.ubuntu == 18
+                  php-version: 7.4
 
             - uses: ./Nominatim/.github/actions/setup-postgresql
               with:
-                  postgresql-version: ${{ matrix.postgresql }}
-                  postgis-version: ${{ matrix.postgis }}
+                  postgresql-version: 13
+                  postgis-version: 3
+
+            - name: Install Postgresql server dev
+              run: sudo apt-get install postgresql-server-dev-13
 
             - uses: ./Nominatim/.github/actions/build-nominatim
               with:
-                  ubuntu: ${{ matrix.ubuntu }}
+                  ubuntu: 20
+                  cmake-args: -DBUILD_MODULE=on
 
             - name: Install test prerequsites
               run: sudo apt-get install -y -qq python3-behave
-              if: matrix.ubuntu == 20
 
-            - name: Install test prerequsites
-              run: pip3 install behave==1.2.6
-              if: matrix.ubuntu == 18
-
-            - name: BDD tests (icu tokenizer)
+            - name: BDD tests (legacy tokenizer)
               run: |
-                  behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=icu --format=progress3
+                  behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy --format=progress3
               working-directory: Nominatim/test/bdd
 
 
@@ -198,7 +177,7 @@ jobs:
 
         strategy:
             matrix:
-                name: [Ubuntu-18, Ubuntu-20, Centos-8]
+                name: [Ubuntu-18, Ubuntu-20, Ubuntu-22]
                 include:
                     - name: Ubuntu-18
                       flavour: ubuntu
@@ -210,9 +189,11 @@ jobs:
                       image: "ubuntu:20.04"
                       ubuntu: 20
                       install_mode: install-apache
-                    - name: Centos-8
-                      flavour: centos
-                      image: "centos:8"
+                    - name: Ubuntu-22
+                      flavour: ubuntu
+                      image: "ubuntu:22.04"
+                      ubuntu: 22
+                      install_mode: install-apache
 
         container:
             image: ${{ matrix.image }}
@@ -251,7 +232,7 @@ jobs:
                 OS: ${{ matrix.name }}
                 INSTALL_MODE: ${{ matrix.install_mode }}
 
-            - uses: actions/download-artifact@v2
+            - uses: actions/download-artifact@v3
               with:
                   name: full-source
                   path: /home/nominatim
@@ -281,6 +262,14 @@ jobs:
               working-directory: /home/nominatim
               if: matrix.flavour == 'centos'
 
+            - name: Print version
+              run: nominatim --version
+              working-directory: /home/nominatim/nominatim-project
+
+            - name: Collect host OS information
+              run: nominatim admin --collect-os-info
+              working-directory: /home/nominatim/nominatim-project
+              
             - name: Import
               run: nominatim import --osm-file ../test.pbf
               working-directory: /home/nominatim/nominatim-project
@@ -309,12 +298,20 @@ jobs:
                   NOMINATIM_REPLICATION_MAX_DIFF=1 nominatim replication --once
               working-directory: /home/nominatim/nominatim-project
 
+            - name: Clean up database
+              run: nominatim refresh --postcodes --word-tokens
+              working-directory: /home/nominatim/nominatim-project
+
             - name: Run reverse-only import
               run : |
                   echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' >> .env
                   nominatim import --osm-file ../test.pbf --reverse-only --no-updates
               working-directory: /home/nominatim/data-env-reverse
 
-            - name: Check reverse import
+            - name: Check reverse-only import
               run: nominatim admin --check-database
               working-directory: /home/nominatim/data-env-reverse
+
+            - name: Clean up database (reverse-only import)
+              run: nominatim refresh --postcodes --word-tokens
+              working-directory: /home/nominatim/nominatim-project

.mypy.ini

@@ -0,0 +1,13 @@
+[mypy]
+
+[mypy-icu.*]
+ignore_missing_imports = True
+
+[mypy-osmium.*]
+ignore_missing_imports = True
+
+[mypy-datrie.*]
+ignore_missing_imports = True
+
+[mypy-dotenv.*]
+ignore_missing_imports = True

.pylintrc

@@ -10,6 +10,9 @@ ignored-modules=icu,datrie
 # closing added here because it sometimes triggers a false positive with
 # 'with' statements.
 ignored-classes=NominatimArgs,closing
-disable=too-few-public-methods,duplicate-code
+# 'too-many-ancestors' is triggered already by deriving from UserDict
+# 'not-context-manager' disabled because it causes false positives once
+#   typed Python is enabled. See also https://github.com/PyCQA/pylint/issues/5273
+disable=too-few-public-methods,duplicate-code,too-many-ancestors,bad-option-value,no-self-use,not-context-manager,use-dict-literal
 
-good-names=i,x,y,fd,db
+good-names=i,x,y,m,fd,db,cc

AUTHORS

@@ -1,15 +1,15 @@
 Nominatim was written by:
 
-  Brian Quinion
-  Sarah Hoffmann
-  Marc Tobias Metten
+* Brian Quinion
+* Sarah Hoffmann
+* Marc Tobias Metten
 
-  markigail
-  gemo1011
-  IrlJidel
-  Frederik Ramm
+* markigail
+* AntoJvlt
+* gemo1011
+* darkshredder
 
 and many more.
 
-For a full list of contributors see
+For a full list of contributors see the Git logs or visit
 https://github.com/openstreetmap/Nominatim/graphs/contributors

CMakeLists.txt

@@ -19,13 +19,24 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 4)
-set(NOMINATIM_VERSION_MINOR 0)
-set(NOMINATIM_VERSION_PATCH 0)
+set(NOMINATIM_VERSION_MINOR 2)
+set(NOMINATIM_VERSION_PATCH 4)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 
 add_definitions(-DNOMINATIM_VERSION="${NOMINATIM_VERSION}")
 
+# Setting GIT_HASH
+find_package(Git)
+if (GIT_FOUND)
+    execute_process(
+        COMMAND "${GIT_EXECUTABLE}" log -1 --format=%h
+        WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+        OUTPUT_VARIABLE GIT_HASH
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_QUIET
+        )
+endif()
 
 #-----------------------------------------------------------------------------
 #  Configuration
@@ -33,7 +44,7 @@ add_definitions(-DNOMINATIM_VERSION="${NOMINATIM_VERSION}")
 
 set(BUILD_IMPORTER on CACHE BOOL "Build everything for importing/updating the database")
 set(BUILD_API on CACHE BOOL "Build everything for the API server")
-set(BUILD_MODULE on CACHE BOOL "Build PostgreSQL module")
+set(BUILD_MODULE off CACHE BOOL "Build PostgreSQL module for legacy tokenizer")
 set(BUILD_TESTS on CACHE BOOL "Build test suite")
 set(BUILD_DOCS on CACHE BOOL "Build documentation")
 set(BUILD_MANPAGE on CACHE BOOL "Build Manual Page")
@@ -52,7 +63,6 @@ if (BUILD_IMPORTER AND BUILD_OSM2PGSQL)
     endif()
     set(BUILD_TESTS_SAVED "${BUILD_TESTS}")
     set(BUILD_TESTS off)
-    set(WITH_LUA off CACHE BOOL "")
     add_subdirectory(osm2pgsql)
     set(BUILD_TESTS ${BUILD_TESTS_SAVED})
 endif()
@@ -226,8 +236,7 @@ if (BUILD_IMPORTER)
             PATTERN __pycache__ EXCLUDE)
     install(DIRECTORY lib-sql DESTINATION ${NOMINATIM_LIBDIR})
 
-    install(FILES data/country_name.sql
-                  ${COUNTRY_GRID_FILE}
+    install(FILES ${COUNTRY_GRID_FILE}
                   data/words.sql
             DESTINATION ${NOMINATIM_DATADIR})
 endif()
@@ -260,12 +269,20 @@ install(FILES settings/env.defaults
               settings/import-address.style
               settings/import-full.style
               settings/import-extratags.style
+              settings/import-admin.lua
+              settings/import-street.lua
+              settings/import-address.lua
+              settings/import-full.lua
+              settings/import-extratags.lua
+              settings/flex-base.lua
               settings/icu_tokenizer.yaml
               settings/country_settings.yaml
         DESTINATION ${NOMINATIM_CONFIGDIR})
 
 install(DIRECTORY settings/icu-rules
         DESTINATION ${NOMINATIM_CONFIGDIR})
+install(DIRECTORY settings/country-names
+        DESTINATION ${NOMINATIM_CONFIGDIR})
 
 if (INSTALL_MUNIN_PLUGINS)
     install(FILES munin/nominatim_importlag

CONTRIBUTING.md

@@ -36,7 +36,7 @@ Nominatim historically hasn't followed a particular coding style but we
 are in process of consolidating the style. The following rules apply:
 
  * Python code uses the official Python style
- * indention
+ * indentation
    * SQL use 2 spaces
    * all other file types use 4 spaces
    * [BSD style](https://en.wikipedia.org/wiki/Indent_style#Allman_style) for braces
@@ -64,3 +64,39 @@ Before submitting a pull request make sure that the tests pass:
   cd build
   make test
 ```
+
+## Releases
+
+Nominatim follows semantic versioning. Major releases are done for large changes
+that require (or at least strongly recommend) a reimport of the databases.
+Minor releases can usually be applied to exisiting databases Patch releases
+contain bug fixes only and are released from a separate branch where the
+relevant changes are cherry-picked from the master branch.
+
+Checklist for releases:
+
+* [ ] increase version in `nominatim/version.py` and CMakeLists.txt
+* [ ] update `ChangeLog` (copy information from patch releases from release branch)
+* [ ] complete `docs/admin/Migration.md`
+* [ ] update EOL dates in `SECURITY.md`
+* [ ] commit and make sure CI tests pass
+* [ ] test migration
+  * download, build and import previous version
+  * migrate using master version
+  * run updates using master version
+* [ ] prepare tarball:
+  * `git clone --recursive https://github.com/osm-search/Nominatim` (switch to right branch!)
+  * `rm -r .git* osm2pgsql/.git*`
+  * copy country data into `data/`
+  * add version to base directory and package
+* [ ] upload tarball to https://nominatim.org
+* [ ] prepare documentation
+  * check out new docs branch
+  * change git checkout instructions to tarball download instructions or adapt version on existing ones
+  * build documentation and copy to https://github.com/osm-search/nominatim-org-site
+  * add new version to history
+* [ ] check release tarball
+  * download tarball as per new documentation instructions
+  * compile and import Nominatim
+  * run `nominatim --version` to confirm correct version
+* [ ] tag new release and add a release on github.com

ChangeLog

@@ -1,3 +1,109 @@
+4.2.4
+ * fix a potential SQL injection in 'nominatim admin --collect-os-info'
+ * fix compatibility issue with PostGIS 3.4
+
+4.2.3
+
+ * fix deletion handling for 'nominatim add-data'
+ * adapt place_force_delete() to new deletion handling
+ * flex style: avoid dropping of postcode areas
+ * fix update errors on address interpolation handling
+
+4.2.2
+
+ * extend flex-style library to fully support all default styles
+ * fix handling of Hebrew aleph
+ * do not assign postcodes to rivers
+ * fix string matching in PHP code
+ * update osm2pgsql (various updates to flex)
+ * fix slow query when deleting places on update
+ * fix CLI details query
+ * fix recalculation of importance values
+ * fix polygon simplification in reverse results
+ * add class/type information to reverse geocodejson result
+ * minor improvements to default tokenizer configuration
+ * various smaller fixes to documentation
+
+4.2.1
+
+ * fix XSS vulnerability in debug view
+
+4.2.0
+
+ * add experimental support for osm2pgsql flex style
+ * introduce secondary importance value to be retrieved from a raster data file
+   (currently still unused, to replace address importance, thanks to @tareqpi)
+ * add new report tool `nominatim admin --collect-os-info`
+   (thanks @micahcochran, @tareqpi)
+ * reorganise index to improve lookup performance and size
+ * run index creation after import in parallel
+ * run ANALYZE more selectively to speed up continuation of indexing
+ * fix crash on update when addr:interpolation receives an illegal value
+ * fix minimum number of retrieved results to be at least 10
+ * fix search for combinations of special term + name (e.g Hotel Bellevue)
+ * do not return interpolations without a parent street on reverse search
+ * improve invalidation of linked places on updates
+ * fix address parsing for interpolation lines
+ * make sure socket timeouts are respected during replication
+   (working around a bug in some versions of pyosmium)
+ * update bundled osm2pgsql to 1.7.1
+ * add support for PostgreSQL 15
+ * typing fixes to work with latest type annotations from typeshed
+ * smaller improvements to documentation (thanks to @mausch)
+
+4.1.1
+
+ * fix XSS vulnerability in debug view
+
+4.1.0
+
+ * switch to ICU tokenizer as default
+ * add housenumber normalization and support optional spaces during search
+ * add postcode format checking and support optional spaces during search
+ * add function for cleaning housenumbers in word table
+ * add updates/deletion of country names imported from OSM
+ * linked places no longer overwrite names from a place permanently
+ * move default country name configuration into yaml file (thanks @tareqpi)
+ * more compact layout for interpolation and TIGER tables
+ * introduce mutations to ICU tokenizer (used for German umlauts)
+ * support reinitializing a full project directory with refresh --website
+ * fix various issues with linked places on updates
+ * add support for external sanitizers and token analyzers
+ * add CLI commands for forced indexing
+ * add CLI command for version report
+ * add offline import mode
+ * change geocodejson to return a feature class in the 'type' field
+ * add ISO3166-2 to address output (thanks @I70l0teN4ik)
+ * improve parsing and matching of addr: tags
+ * support relations as street members of associatedStreet
+ * better ranking for address results from TIGER data
+ * adapt rank classification to changed tag usage in OSM
+ * update bundled osm2pgsql to 1.6.0
+ * add typing information to Python code
+ * improve unit test coverage
+ * reorganise and speed up code for BDD tests, drop support for scenes
+ * move PHP unit tests to PHP 9.5
+ * extensive typo fixes in documentation (thanks @woodpeck,@StephanGeorg,
+   @amandasaurus, @nslxndr, @stefkiourk, @Luflosi, @kianmeng)
+ * drop official support for installation on CentOS
+ * add installation instructions for Ubuntu 22.04
+ * add support for PHP8
+ * add setup instructions for updates and systemd
+ * drop support for PostgreSQL 9.5
+
+4.0.2
+
+ * fix XSS vulnerability in debug view
+
+4.0.1
+
+ * fix initialisation error in replication script
+ * ICU tokenizer: avoid any special characters in word tokens
+ * better error message when API php script does not exist
+ * fix quoting of house numbers in SQL queries
+ * small fixes and improvements in search query parsing
+ * add documentation for moving the database to a different machine
+
 4.0.0
 
  * refactor name token computation and introduce ICU tokenizer
@@ -27,6 +133,10 @@
  * add testing of installation scripts via CI
  * drop support for Python < 3.6 and Postgresql < 9.5
 
+3.7.3
+
+ * fix XSS vulnerability in debug view
+
 3.7.2
 
  * fix database check for reverse-only imports

SECURITY.md

@@ -9,10 +9,11 @@ versions.
 
 | Version | End of support for security updates |
 | ------- | ----------------------------------- |
+| 4.2.x   | 2024-11-24                          |
+| 4.1.x   | 2024-08-05                          |
+| 4.0.x   | 2023-11-02                          |
 | 3.7.x   | 2023-04-05                          |
 | 3.6.x   | 2022-12-12                          |
-| 3.5.x   | 2022-06-05                          |
-| 3.4.x   | 2021-10-24                          |
 
 ## Reporting a Vulnerability
 

VAGRANT.md

@@ -1,6 +1,6 @@
 # Install Nominatim in a virtual machine for development and testing
 
-This document describes how you can install Nominatim inside a Ubuntu 16
+This document describes how you can install Nominatim inside a Ubuntu 22
 virtual machine on your desktop/laptop (host machine). The goal is to give
 you a development environment to easily edit code and run the test suite
 without affecting the rest of your system. 
@@ -42,9 +42,9 @@ is.
 
       ```
       # inside the virtual machine:
-      cd build
-      wget --no-verbose --output-document=/tmp/monaco.osm.pbf http://download.geofabrik.de/europe/monaco-latest.osm.pbf
-      ./utils/setup.php --osm-file /tmp/monaco.osm.pbf --osm2pgsql-cache 1000 --all 2>&1 | tee monaco.$$.log
+      cd nominatim-project
+      wget --no-verbose --output-document=monaco.osm.pbf http://download.geofabrik.de/europe/monaco-latest.osm.pbf
+      nominatim import --osm-file monaco.osm.pbf 2>&1 | tee monaco.$$.log
       ```
 
     To repeat an import you'd need to delete the database first
@@ -56,7 +56,7 @@ is.
 ## Development
 
 Vagrant maps the virtual machine's port 8089 to your host machine. Thus you can
-see Nominatim in action on [locahost:8089](http://localhost:8089/nominatim/).
+see Nominatim in action on [localhost:8089](http://localhost:8089/nominatim/).
 
 You edit code on your host machine in any editor you like. There is no need to
 restart any software: just refresh your browser window.
@@ -69,8 +69,7 @@ installation.
 PHP errors are written to `/var/log/apache2/error.log`.
 
 With `echo` and `var_dump()` you write into the output (HTML/XML/JSON) when
-you either add `&debug=1` to the URL (preferred) or set
-`@define('CONST_Debug', true);` in `settings/local.php`.
+you either add `&debug=1` to the URL.
 
 In the Python BDD test you can use `logger.info()` for temporary debug
 statements.
@@ -130,6 +129,10 @@ and then
 Yes, Vagrant and Virtualbox can be installed on MS Windows just fine. You need a 64bit
 version of Windows.
 
+##### Will it run on Apple Silicon?
+
+You might need to replace Virtualbox with [Parallels](https://www.parallels.com/products/desktop/).
+There is no free/open source version of Parallels.
 
 ##### Why Monaco, can I use another country?
 
@@ -141,11 +144,12 @@ No. Long running Nominatim installations will differ once new import features (o
 bug fixes) get added since those usually only get applied to new/changed data.
 
 Also this document skips the optional Wikipedia data import which affects ranking
-of search results. See [Nominatim installation](https://nominatim.org/release-docs/latest/admin/Installation) for details.
+of search results. See [Nominatim installation](https://nominatim.org/release-docs/latest/admin/Installation)
+for details.
 
 ##### Why Ubuntu? Can I test CentOS/Fedora/CoreOS/FreeBSD?
 
-There is a Vagrant script for CentOS available, but the Nominatim directory
+There used to be a Vagrant script for CentOS available, but the Nominatim directory
 isn't symlinked/mounted to the host which makes development trickier. We used
 it mainly for debugging installation with SELinux.
 
@@ -154,13 +158,16 @@ are slightly different, e.g. the name of the package manager, Apache2 package
 name, location of files. We chose Ubuntu because that is closest to the
 nominatim.openstreetmap.org production environment.
 
-You can configure/download other Vagrant boxes from [https://app.vagrantup.com/boxes/search](https://app.vagrantup.com/boxes/search).
+You can configure/download other Vagrant boxes from
+[https://app.vagrantup.com/boxes/search](https://app.vagrantup.com/boxes/search).
 
 ##### How can I connect to an existing database?
 
-Let's say you have a Postgres database named `nominatim_it` on server `your-server.com` and port `5432`. The Postgres username is `postgres`. You can edit `settings/local.php` and point Nominatim to it.
+Let's say you have a Postgres database named `nominatim_it` on server `your-server.com`
+and port `5432`. The Postgres username is `postgres`. You can edit the `.env` in your
+project directory and point Nominatim to it.
 
-    pgsql:host=your-server.com;port=5432;user=postgres;dbname=nominatim_it
+    NOMINATIM_DATABASE_DSN="pgsql:host=your-server.com;port=5432;user=postgres;dbname=nominatim_it
 
 No data import or restarting necessary.
 
@@ -169,11 +176,12 @@ If the Postgres installation is behind a firewall, you can try
     ssh -L 9999:localhost:5432 your-username@your-server.com
 
 inside the virtual machine. It will map the port to `localhost:9999` and then
-you edit `settings/local.php` with
+you edit `.env` file with
 
-    @define('CONST_Database_DSN', 'pgsql:host=localhost;port=9999;user=postgres;dbname=nominatim_it');
+    NOMINATIM_DATABASE_DSN="pgsql:host=localhost;port=9999;user=postgres;dbname=nominatim_it"
 
-To access postgres directly remember to specify the hostname, e.g. `psql --host localhost --port 9999 nominatim_it`
+To access postgres directly remember to specify the hostname,
+e.g. `psql --host localhost --port 9999 nominatim_it`
 
 
 ##### My computer is slow and the import takes too long. Can I start the virtual machine "in the cloud"?

cmake/tool-installed.tmpl

@@ -7,6 +7,9 @@ sys.path.insert(1, '@NOMINATIM_LIBDIR@/lib-python')
 os.environ['NOMINATIM_NOMINATIM_TOOL'] = os.path.abspath(__file__)
 
 from nominatim import cli
+from nominatim import version
+
+version.GIT_COMMIT_HASH = '@GIT_HASH@'
 
 exit(cli.nominatim(module_dir='@NOMINATIM_LIBDIR@/module',
                    osm2pgsql_path='@NOMINATIM_LIBDIR@/osm2pgsql',

cmake/tool.tmpl

@@ -7,6 +7,9 @@ sys.path.insert(1, '@CMAKE_SOURCE_DIR@')
 os.environ['NOMINATIM_NOMINATIM_TOOL'] = os.path.abspath(__file__)
 
 from nominatim import cli
+from nominatim import version
+
+version.GIT_COMMIT_HASH = '@GIT_HASH@'
 
 exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module',
                    osm2pgsql_path='@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql',

codecov.yml

@@ -1,14 +0,0 @@
-codecov:
-  require_ci_to_pass: yes
-
-coverage:
-  status:
-    project: off
-    patch: off
-
-comment:
-  require_changes: true
-  after_n_builds: 2
-
-fixes:
-  - "Nominatim/::"

docs/CMakeLists.txt

@@ -23,10 +23,9 @@ foreach (src ${DOC_SOURCES})
 endforeach()
 
 ADD_CUSTOM_TARGET(doc
-   COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Centos-7.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Centos-7.md
-   COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Centos-8.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Centos-8.md
    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-18.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-18.md
    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-20.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-20.md
+   COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-22.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-22.md
    COMMAND PYTHONPATH=${PROJECT_SOURCE_DIR} mkdocs build -d ${CMAKE_CURRENT_BINARY_DIR}/../site-html -f ${CMAKE_CURRENT_BINARY_DIR}/../mkdocs.yml
 )
 

docs/admin/Advanced-Installations.md

@@ -101,7 +101,7 @@ This will get diffs from the replication server, import diffs and index
 the database. The default replication server in the
 script([Geofabrik](https://download.geofabrik.de)) provides daily updates.
 
-## Importing Nominatim to an external PostgreSQL database
+## Using an external PostgreSQL database
 
 You can install Nominatim using a database that runs on a different server when
 you have physical access to the file system on the other server. Nominatim
@@ -109,6 +109,11 @@ uses a custom normalization library that needs to be made accessible to the
 PostgreSQL server. This section explains how to set up the normalization
 library.
 
+!!! note
+    The external module is only needed when using the legacy tokenizer.
+    If you have chosen the ICU tokenizer, then you can ignore this section
+    and follow the standard import documentation.
+
 ### Option 1: Compiling the library on the database server
 
 The most sure way to get a working library is to compile it on the database
@@ -167,3 +172,45 @@ NOMINATIM_DATABASE_MODULE_PATH="<directory on the database server where nominati
 
 Now change the `NOMINATIM_DATABASE_DSN` to point to your remote server and continue
 to follow the [standard instructions for importing](Import.md).
+
+
+## Moving the database to another machine
+
+For some configurations it may be useful to run the import on one machine, then
+move the database to another machine and run the Nominatim service from there.
+For example, you might want to use a large machine to be able to run the import
+quickly but only want a smaller machine for production because there is not so
+much load. Or you might want to do the import once and then replicate the
+database to many machines.
+
+The important thing to keep in mind when transferring the Nominatim installation
+is that you need to transfer the database _and the project directory_. Both
+parts are essential for your installation.
+
+The Nominatim database can be transferred using the `pg_dump`/`pg_restore` tool.
+Make sure to use the same version of PostgreSQL and PostGIS on source and
+target machine.
+
+!!! note
+    Before creating a dump of your Nominatim database, consider running
+    `nominatim freeze` first. Your database looses the ability to receive further
+    data updates but the resulting database is only about a third of the size
+    of a full database.
+
+Next install Nominatim on the target machine by following the standard installation
+instructions. Again, make sure to use the same version as the source machine.
+
+Create a project directory on your destination machine and set up the `.env`
+file to match the configuration on the source machine. Finally run
+
+    nominatim refresh --website
+
+to make sure that the local installation of Nominatim will be used.
+
+If you are using the legacy tokenizer you might also have to switch to the
+PostgreSQL module that was compiled on your target machine. If you get errors
+that PostgreSQL cannot find or access `nominatim.so` then rerun
+
+   nominatim refresh --functions
+
+on the target machine to update the the location of the module.

docs/admin/Deployment.md

@@ -82,7 +82,7 @@ The website should now be available on `http://localhost/nominatim`.
 
 ### Installing the required packages
 
-Nginx has no built-in PHP interpreter. You need to use php-fpm as a deamon for
+Nginx has no built-in PHP interpreter. You need to use php-fpm as a daemon for
 serving PHP cgi.
 
 On Ubuntu/Debian install nginx and php-fpm with:
@@ -99,7 +99,7 @@ Unix socket instead, change the pool configuration
 
 ``` ini
 ; Replace the tcp listener and add the unix socket
-listen = /var/run/php-fpm.sock
+listen = /var/run/php-fpm-nominatim.sock
 
 ; Ensure that the daemon runs as the correct user
 listen.owner = www-data
@@ -121,7 +121,7 @@ location @php {
     fastcgi_param SCRIPT_FILENAME "$document_root$uri.php";
     fastcgi_param PATH_TRANSLATED "$document_root$uri.php";
     fastcgi_param QUERY_STRING    $args;
-    fastcgi_pass unix:/var/run/php-fpm.sock;
+    fastcgi_pass unix:/var/run/php-fpm-nominatim.sock;
     fastcgi_index index.php;
     include fastcgi_params;
 }
@@ -131,7 +131,7 @@ location ~ [^/]\.php(/|$) {
     if (!-f $document_root$fastcgi_script_name) {
         return 404;
     }
-    fastcgi_pass unix:/var/run/php-fpm.sock;
+    fastcgi_pass unix:/var/run/php-fpm-nominatim.sock;
     fastcgi_index search.php;
     include fastcgi.conf;
 }
@@ -140,3 +140,9 @@ location ~ [^/]\.php(/|$) {
 Restart the nginx and php-fpm services and the website should now be available
 at `http://localhost/`.
 
+## Nominatim with other webservers
+
+Users have created instructions for other webservers:
+
+* [Caddy](https://github.com/osm-search/Nominatim/discussions/2580)
+

docs/admin/Faq.md

@@ -79,7 +79,7 @@ When running the import you may get a version mismatch:
 
 pg_config seems to use bad includes sometimes when multiple versions
 of PostgreSQL are available in the system. Make sure you remove the
-server development libraries (`postgresql-server-dev-9.5` on Ubuntu)
+server development libraries (`postgresql-server-dev-13` on Ubuntu)
 and recompile (`cmake .. && make`).
 
 
@@ -106,11 +106,6 @@ If you are using a flatnode file, then it may also be that the underlying
 filesystem does not fully support 'mmap'. A notable candidate is virtualbox's
 vboxfs.
 
-### I see the error: "clang: Command not found" on CentOS
-
-On CentOS 7 users reported `/opt/rh/llvm-toolset-7/root/usr/bin/clang: Command not found`.
-Double-check clang is installed. Instead of `make` try running `make CLANG=true`.
-
 ### nominatim UPDATE failed: ERROR: buffer 179261 is not owned by resource owner Portal
 
 Several users [reported this](https://github.com/openstreetmap/Nominatim/issues/1168)
@@ -126,22 +121,6 @@ The server cannot access your database. Add `&debug=1` to your URL
 to get the full error message.
 
 
-### On CentOS the website shows "Could not connect to server"
-
-`could not connect to server: No such file or directory`
-
-On CentOS v7 the PostgreSQL server is started with `systemd`. Check if
-`/usr/lib/systemd/system/httpd.service` contains a line `PrivateTmp=true`. If
-so then Apache cannot see the `/tmp/.s.PGSQL.5432` file. It's a good security
-feature, so use the
-[preferred solution](../appendix/Install-on-Centos-7.md#adding-selinux-security-settings).
-
-However, you can solve this the quick and dirty way by commenting out that line and then run
-
-    sudo systemctl daemon-reload
-    sudo systemctl restart httpd
-
-
 ### Website reports "DB Error: insufficient permissions"
 
 The user the webserver, e.g. Apache, runs under needs to have access to the
@@ -181,9 +160,6 @@ by everybody, e.g.
 
 Try `chmod a+r nominatim.so; chmod a+x nominatim.so`.
 
-When running SELinux, make sure that the
-[context is set up correctly](../appendix/Install-on-Centos-7.md#adding-selinux-security-settings).
-
 When you recently updated your operating system, updated PostgreSQL to
 a new version or moved files (e.g. the build directory) you should
 recreate `nominatim.so`. Try

docs/admin/Import.md

@@ -74,15 +74,15 @@ but it will improve the quality of the results if this is installed.
 This data is available as a binary download. Put it into your project directory:
 
     cd $PROJECT_DIR
-    wget https://www.nominatim.org/data/wikimedia-importance.sql.gz
+    wget https://nominatim.org/data/wikimedia-importance.sql.gz
 
 The file is about 400MB and adds around 4GB to the Nominatim database.
 
 !!! tip
-    If you forgot to download the wikipedia rankings, you can also add
-    importances after the import. Download the files, then run
-    `nominatim refresh --wiki-data --importance`. Updating importances for
-    a planet can take a couple of hours.
+    If you forgot to download the wikipedia rankings, then you can
+    also add importances after the import. Download the SQL files, then
+    run `nominatim refresh --wiki-data --importance`. Updating
+    importances for a planet will take a couple of hours.
 
 ### External postcodes
 
@@ -92,8 +92,8 @@ and the UK (using the [CodePoint OpenData set](https://osdatahub.os.uk/downloads
 This data can be optionally downloaded into the project directory:
 
     cd $PROJECT_DIR
-    wget https://www.nominatim.org/data/gb_postcodes.csv.gz
-    wget https://www.nominatim.org/data/us_postcodes.csv.gz
+    wget https://nominatim.org/data/gb_postcodes.csv.gz
+    wget https://nominatim.org/data/us_postcodes.csv.gz
 
 You can also add your own custom postcode sources, see
 [Customization of postcodes](../customize/Postcodes.md).
@@ -139,7 +139,7 @@ import. So this option is particularly interesting if you plan to transfer the
 database or reuse the space later.
 
 !!! warning
-    The datastructure for updates are also required when adding additional data
+    The data structure for updates are also required when adding additional data
     after the import, for example [TIGER housenumber data](../customize/Tiger.md).
     If you plan to use those, you must not use the `--no-updates` parameter.
     Do a normal import, add the external data and once you are done with
@@ -150,7 +150,7 @@ database or reuse the space later.
 
 If you only want to use the Nominatim database for reverse lookups or
 if you plan to use the installation only for exports to a
-[photon](https://photon.komoot.de/) database, then you can set up a database
+[photon](https://photon.komoot.io/) database, then you can set up a database
 without search indexes. Add `--reverse-only` to your setup command above.
 
 This saves about 5% of disk space.

docs/admin/Installation.md

@@ -4,10 +4,9 @@ This page contains generic installation instructions for Nominatim and its
 prerequisites. There are also step-by-step instructions available for
 the following operating systems:
 
+  * [Ubuntu 22.04](../appendix/Install-on-Ubuntu-22.md)
   * [Ubuntu 20.04](../appendix/Install-on-Ubuntu-20.md)
   * [Ubuntu 18.04](../appendix/Install-on-Ubuntu-18.md)
-  * [CentOS 8](../appendix/Install-on-Centos-8.md)
-  * [CentOS 7.2](../appendix/Install-on-Centos-7.md)
 
 These OS-specific instructions can also be found in executable form
 in the `vagrant/` directory.
@@ -25,8 +24,9 @@ and can't offer support.
 ### Software
 
 !!! Warning
-    For larger installations you **must have** PostgreSQL 11+ and Postgis 3+
+    For larger installations you **must have** PostgreSQL 11+ and PostGIS 3+
     otherwise import and queries will be slow to the point of being unusable.
+    Query performance has marked improvements with PostgreSQL 13+ and PostGIS 3.2+.
 
 For compiling:
 
@@ -42,7 +42,7 @@ For compiling:
 
 For running Nominatim:
 
-  * [PostgreSQL](https://www.postgresql.org) (9.5+ will work, 11+ strongly recommended)
+  * [PostgreSQL](https://www.postgresql.org) (9.6+ will work, 11+ strongly recommended)
   * [PostGIS](https://postgis.net) (2.2+ will work, 3.0+ strongly recommended)
   * [Python 3](https://www.python.org/) (3.6+)
   * [Psycopg2](https://www.psycopg.org) (2.7+)
@@ -67,10 +67,10 @@ the [Development section](../develop/Development-Environment.md).
 ### Hardware
 
 A minimum of 2GB of RAM is required or installation will fail. For a full
-planet import 64GB of RAM or more are strongly recommended. Do not report
+planet import 128GB of RAM or more are strongly recommended. Do not report
 out of memory problems if you have less than 64GB RAM.
 
-For a full planet install you will need at least 900GB of hard disk space.
+For a full planet install you will need at least 1TB of hard disk space.
 Take into account that the OSM database is growing fast.
 Fast disks are essential. Using NVME disks is recommended.
 
@@ -89,8 +89,7 @@ your `postgresql.conf` file.
     work_mem = (50MB)
     effective_cache_size = (24GB)
     synchronous_commit = off
-    checkpoint_segments = 100 # only for postgresql <= 9.4
-    max_wal_size = 1GB # postgresql > 9.4
+    max_wal_size = 1GB
     checkpoint_timeout = 10min
     checkpoint_completion_target = 0.9
 
@@ -113,7 +112,7 @@ For the initial import, you should also set:
     fsync = off
     full_page_writes = off
 
-Don't forget to reenable them after the initial import or you risk database
+Don't forget to re-enable them after the initial import or you risk database
 corruption.
 
 
@@ -130,13 +129,13 @@ If you want to install latest development version from github, make sure to
 also check out the osm2pgsql subproject:
 
 ```
-git clone --recursive git://github.com/openstreetmap/Nominatim.git
+git clone --recursive https://github.com/openstreetmap/Nominatim.git
 ```
 
 The development version does not include the country grid. Download it separately:
 
 ```
-wget -O Nominatim/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
+wget -O Nominatim/data/country_osm_grid.sql.gz https://nominatim.org/data/country_grid.sql.gz
 ```
 
 ### Building Nominatim
@@ -158,6 +157,17 @@ make
 sudo make install
 ```
 
+!!! warning
+    The default installation no longer compiles the PostgreSQL module that
+    is needed for the legacy tokenizer from older Nominatim versions. If you
+    are upgrading an older database or want to run the
+    [legacy tokenizer](../customize/Tokenizers.md#legacy-tokenizer) for
+    some other reason, you need to enable the PostgreSQL module via
+    cmake: `cmake -DBUILD_MODULE=on ../Nominatim`. To compile the module
+    you need to have the server development headers for PostgreSQL installed.
+    On Ubuntu/Debian run: `sudo apt install postgresql-server-dev-<postgresql version>`
+
+
 Nominatim installs itself into `/usr/local` per default. To choose a different
 installation directory add `-DCMAKE_INSTALL_PREFIX=<install root>` to the
 cmake command. Make sure that the `bin` directory is available in your path

docs/admin/Maintenance.md

@@ -34,6 +34,30 @@ to rerun the statistics computation when adding larger amounts of new data,
 for example, when adding an additional country via `nominatim add-data`.
 
 
+## Forcing recomputation of places and areas
+
+Command: `nominatim refresh --data-object [NWR]<id> --data-area [NWR]<id>`
+
+When running replication updates, Nominatim tries to recompute the search
+and address information for all places that are affected by a change. But it
+needs to restrict the total number of changes to make sure it can keep up
+with the minutely updates. Therefore it will refrain from propagating changes
+that affect a lot of objects.
+
+The administrator may force an update of places in the database.
+`nominatim refresh --data-object` invalidates a single OSM object.
+`nominatim refresh --data-area` invalidates an OSM object and all dependent
+objects. That are usually the places that inside its area or around the
+center of the object. Both commands expect the OSM object as an argument
+of the form OSM type + OSM id. The type must be `N` (node), `W` (way) or
+`R` (relation).
+
+After invalidating the object, indexing must be run again. If continuous
+update are running in the background, the objects will be recomputed together
+with the next round of updates. Otherwise you need to run `nominatim index`
+to finish the recomputation.
+
+
 ## Removing large deleted objects
 
 Nominatim refuses to delete very large areas because often these deletions are

docs/admin/Migration.md

@@ -15,6 +15,23 @@ breaking changes. **Please read them before running the migration.**
     If you are migrating from a version <3.6, then you still have to follow
     the manual migration steps up to 3.6.
 
+## 4.0.0 -> 4.1.0
+
+### ICU tokenizer is the new default
+
+Nominatim now installs the [ICU tokenizer](../customize/Tokenizers.md#icu-tokenizer)
+by default. This only has an effect on newly installed databases. When
+updating older databases, it keeps its installed tokenizer. If you still
+run with the legacy tokenizer, make sure to compile Nominatim with the
+PostgreSQL module, see [Installation](Installation.md#building-nominatim).
+
+### geocodejson output changed
+
+The `type` field of the geocodejson output has changed. It now contains
+the address class of the object instead of the value of the OSM tag. If
+your client has used the `type` field, switch them to read `osm_value`
+instead.
+
 ## 3.7.0 -> 4.0.0
 
 ### NOMINATIM_PHRASE_CONFIG removed

docs/admin/Setup-Nominatim-UI.md

@@ -16,12 +16,12 @@ and run it. Grab the latest release from
 [nominatim-ui's Github release page](https://github.com/osm-search/nominatim-ui/releases)
 and unpack it. You can use `nominatim-ui-x.x.x.tar.gz` or `nominatim-ui-x.x.x.zip`.
 
-Next you need to adapt the UI yo your installation. Custom settings need to be
+Next you need to adapt the UI to your installation. Custom settings need to be
 put into `dist/theme/config.theme.js`. At a minimum you need to
 set `Nominatim_API_Endpoint` to point to your Nominatim installation:
 
     cd nominatim-ui
-    echo "Nominatim_Config.Nominatim_API_Endpoint='https:\\myserver.org\nominatim';" > dist/theme/config.theme.js
+    echo "Nominatim_Config.Nominatim_API_Endpoint='https://myserver.org/nominatim/';" > dist/theme/config.theme.js
 
 For the full set of available settings, have a look at `dist/config.defaults.js`.
 
@@ -161,24 +161,16 @@ directory like this:
   # If no endpoint is given, then use search.
   RewriteRule ^(/|$)   "search.php"
 
-  # If format-html is explicity requested, forward to the UI.
+  # If format-html is explicitly requested, forward to the UI.
   RewriteCond %{QUERY_STRING} "format=html"
-  RewriteRule ^([^/]+).php ui/$1.html [R,END]
-  # Same but .php suffix is missing.
-  RewriteCond %{QUERY_STRING} "format=html"
-  RewriteRule ^([^/]+) ui/$1.html [R,END]
+  RewriteRule ^([^/]+)(.php)? ui/$1.html [R,END]
 
   # If no format parameter is there then forward anything
   # but /reverse and /lookup to the UI.
   RewriteCond %{QUERY_STRING} "!format="
   RewriteCond %{REQUEST_URI}  "!/lookup"
   RewriteCond %{REQUEST_URI}  "!/reverse"
-  RewriteRule ^([^/]+).php ui/$1.html [R,END]
-  # Same but .php suffix is missing.
-  RewriteCond %{QUERY_STRING} "!format="
-  RewriteCond %{REQUEST_URI}  "!/lookup"
-  RewriteCond %{REQUEST_URI}  "!/reverse"
-  RewriteRule ^([^/]+) ui/$1.html [R,END]
+  RewriteRule ^([^/]+)(.php)? ui/$1.html [R,END]
 </Directory>
 ```
 

docs/admin/Update.md

@@ -70,7 +70,7 @@ The update application keeps running forever and retrieves and applies
 new updates from the server as they are published.
 
 You can run this command as a simple systemd service. Create a service
-description like that in `/etc/systemd/system/nominatim-update.service`:
+description like that in `/etc/systemd/system/nominatim-updates.service`:
 
 ```
 [Unit]
@@ -122,14 +122,71 @@ cd /srv/nominatim
 
 while true; do
   nominatim replication --once
-  if [ -f "/srv/nominatim/schedule-mainenance" ]; then
-    rm /srv/nominatim/schedule-mainenance
+  if [ -f "/srv/nominatim/schedule-maintenance" ]; then
+    rm /srv/nominatim/schedule-maintenance
     nominatim refresh --postcodes
   fi
 done
 ```
 
-A cron job then creates the file `/srv/nominatim/need-mainenance` once per night.
+A cron job then creates the file `/srv/nominatim/schedule-maintenance` once per night.
+
+##### One-time mode with systemd
+
+You can run the one-time mode with a systemd timer & service.
+
+Create a timer description like `/etc/systemd/system/nominatim-updates.timer`:
+
+```
+[Unit]
+Description=Timer to start updates of Nominatim
+
+[Timer]
+OnActiveSec=2
+OnUnitActiveSec=1min
+Unit=nominatim-updates.service
+
+[Install]
+WantedBy=multi-user.target
+```
+
+And then a similar service definition: `/etc/systemd/system/nominatim-updates.service`:
+
+```
+[Unit]
+Description=Single updates of Nominatim
+
+[Service]
+WorkingDirectory=/srv/nominatim
+ExecStart=nominatim replication --once
+StandardOutput=append:/var/log/nominatim-updates.log
+StandardError=append:/var/log/nominatim-updates.error.log
+User=nominatim
+Group=nominatim
+Type=simple
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Replace the `WorkingDirectory` with your project directory. Also adapt user and
+group names as required. `OnUnitActiveSec` defines how often the individual
+update command is run.
+
+Now activate the service and start the updates:
+
+```
+sudo systemctl daemon-reload
+sudo systemctl enable nominatim-updates.timer
+sudo systemctl start nominatim-updates.timer
+```
+
+You can stop future data updates, while allowing any current, in-progress
+update steps to finish, by running `sudo systemctl stop
+nominatim-updates.timer` and waiting until `nominatim-updates.service` isn't
+running (`sudo systemctl is-active nominatim-updates.service`). Current output
+from the update can be seen like above (`systemctl status
+nominatim-updates.service`).
 
 
 #### Catch-up mode
@@ -158,7 +215,7 @@ replication catch-up at whatever interval you desire.
     a replication source with an update frequency that is an order of magnitude
     lower. For example, if you want to update once a day, use an hourly updated
     source. This makes sure that you don't miss an entire day of updates when
-    the source is unexpectely late to publish its update.
+    the source is unexpectedly late to publish its update.
 
     If you want to use the source with the same update frequency (e.g. a daily
     updated source with daily updates), use the

docs/api/Faq.md

@@ -59,3 +59,27 @@ suited for these kinds of queries.
 
 That said if you installed your own Nominatim instance you can use the
 `nominatim export` PHP script as basis to return such lists.
+
+#### 7. My result has a wrong postcode. Where does it come from?
+
+Most places in OSM don't have a postcode, so Nominatim tries to interpolate
+one. It first look at all the places that make up the address of the place.
+If one of them has a postcode defined, this is the one to be used. When
+none of the address parts has a postcode either, Nominatim interpolates one
+from the surrounding objects. If the postcode is for your result is one, then
+most of the time there is an OSM object with the wrong postcode nearby.
+
+To find the bad postcode, go to
+[https://nominatim.openstreetmap.org](https://nominatim.openstreetmap.org)
+and search for your place. When you have found it, click on the 'details' link
+under the result to go to the details page. There is a field 'Computed Postcode'
+which should display the bad postcode. Click on the 'how?' link. A small
+explanation text appears. It contains a link to a query for Overpass Turbo.
+Click on that and you get a map with all places in the area that have the bad
+postcode. If none is displayed, zoom the map out a bit and then click on 'Run'.
+
+Now go to [OpenStreetMap](https://openstreetmap.org) and fix the error you
+have just found. It will take at least a day for Nominatim to catch up with
+your data fix. Sometimes longer, depending on how much editing activity is in
+the area.
+

docs/api/Lookup.md

@@ -90,11 +90,11 @@ This overrides the specified machine readable format. (Default: 0)
 
 ##### XML
 
-[https://nominatim.openstreetmap.org/lookup?osm_ids=R146656,W104393803,N240109189](https://nominatim.openstreetmap.org/lookup?osm_ids=R146656,W104393803,N240109189)
+[https://nominatim.openstreetmap.org/lookup?osm_ids=R146656,W104393803,N240109189](https://nominatim.openstreetmap.org/lookup?osm_ids=R146656,W50637691,N240109189)
 
 ```xml
-  <lookupresults timestamp="Mon, 29 Jun 15 18:01:33 +0000" attribution="Data © OpenStreetMap contributors, ODbL 1.0. https://www.openstreetmap.org/copyright" querystring="R146656,W104393803,N240109189" polygon="false">
-    <place place_id="127761056" osm_type="relation" osm_id="146656" place_rank="16" lat="53.4791466" lon="-2.2447445" display_name="Manchester, Greater Manchester, North West England, England, United Kingdom" class="boundary" type="administrative" importance="0.704893333438333">
+  <lookupresults timestamp="Mon, 28 Mar 22 14:38:54 +0000" attribution="Data &#xA9; OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright" querystring="R146656,W50637691,N240109189" more_url="">
+    <place place_id="282236157" osm_type="relation" osm_id="146656" place_rank="16" address_rank="16" boundingbox="53.3401044,53.5445923,-2.3199185,-2.1468288" lat="53.44246175" lon="-2.2324547359718547" display_name="Manchester, Greater Manchester, North West England, England, United Kingdom" class="boundary" type="administrative" importance="0.35">
       <city>Manchester</city>
       <county>Greater Manchester</county>
       <state_district>North West England</state_district>
@@ -102,21 +102,20 @@ This overrides the specified machine readable format. (Default: 0)
       <country>United Kingdom</country>
       <country_code>gb</country_code>
     </place>
-    <place place_id="77769745" osm_type="way" osm_id="104393803" place_rank="30" lat="52.5162024" lon="13.3777343363579" display_name="Brandenburg Gate, 1, Pariser Platz, Mitte, Berlin, 10117, Germany" class="tourism" type="attraction" importance="0.443472858361592">
-      <attraction>Brandenburg Gate</attraction>
-      <house_number>1</house_number>
-      <pedestrian>Pariser Platz</pedestrian>
-      <suburb>Mitte</suburb>
-      <city_district>Mitte</city_district>
-      <city>Berlin</city>
-      <state>Berlin</state>
-      <postcode>10117</postcode>
+    <place place_id="115462561" osm_type="way" osm_id="50637691" place_rank="30" address_rank="30" boundingbox="52.3994612,52.3996426,13.0479574,13.0481754" lat="52.399550700000006" lon="13.048066846939687" display_name="Brandenburger Tor, Brandenburger Stra&#xDF;e, Historische Innenstadt, Innenstadt, Potsdam, Brandenburg, 14467, Germany" class="tourism" type="attraction" importance="0.29402874005524">
+      <tourism>Brandenburger Tor</tourism>
+      <road>Brandenburger Stra&#xDF;e</road>
+      <suburb>Historische Innenstadt</suburb>
+      <city>Potsdam</city>
+      <state>Brandenburg</state>
+      <postcode>14467</postcode>
       <country>Germany</country>
       <country_code>de</country_code>
     </place>
-    <place place_id="2570600569" osm_type="node" osm_id="240109189" place_rank="15" lat="52.5170365" lon="13.3888599" display_name="Berlin, Germany" class="place" type="city" importance="0.822149797630868">
+    <place place_id="567505" osm_type="node" osm_id="240109189" place_rank="15" address_rank="16" boundingbox="52.3586925,52.6786925,13.2396024,13.5596024" lat="52.5186925" lon="13.3996024" display_name="Berlin, 10178, Germany" class="place" type="city" importance="0.78753902824914">
       <city>Berlin</city>
       <state>Berlin</state>
+      <postcode>10178</postcode>
       <country>Germany</country>
       <country_code>de</country_code>
     </place>
@@ -125,38 +124,50 @@ This overrides the specified machine readable format. (Default: 0)
 
 ##### JSON with extratags
 
-[https://nominatim.openstreetmap.org/lookup?osm_ids=W50637691&format=json](https://nominatim.openstreetmap.org/lookup?osm_ids=W50637691&format=json)
+[https://nominatim.openstreetmap.org/lookup?osm_ids=W50637691&format=json&extratags=1](https://nominatim.openstreetmap.org/lookup?osm_ids=W50637691&format=json&extratags=1)
 
 ```json
 [
-  {
-    "place_id": "84271358",
-    "licence": "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright",
-    "osm_type": "way",
-    "osm_id": "50637691",
-    "lat": "52.39955055",
-    "lon": "13.04806574678",
-    "display_name": "Brandenburger Tor, Brandenburger Straße, Nördliche Innenstadt, Innenstadt, Potsdam, Brandenburg, 14467, Germany",
-    "class": "historic",
-    "type": "city_gate",
-    "importance": "0.221233780277011",
-    "address": {
-      "address29": "Brandenburger Tor",
-      "pedestrian": "Brandenburger Straße",
-      "suburb": "Nördliche Innenstadt",
-      "city": "Potsdam",
-      "state": "Brandenburg",
-      "postcode": "14467",
-      "country": "Germany",
-      "country_code": "de"
-    },
-    "extratags": {
-      "image": "http://commons.wikimedia.org/wiki/File:Potsdam_brandenburger_tor.jpg",
-      "wikidata": "Q695045",
-      "wikipedia": "de:Brandenburger Tor (Potsdam)",
-      "wheelchair": "yes",
-      "description": "Kleines Brandenburger Tor in Potsdam"
-    }
-  }
+   {
+      "place_id": 115462561,
+      "licence": "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright",
+      "osm_type": "way",
+      "osm_id": 50637691,
+      "boundingbox": [
+        "52.3994612",
+        "52.3996426",
+        "13.0479574",
+        "13.0481754"
+      ],
+      "lat": "52.399550700000006",
+      "lon": "13.048066846939687",
+      "display_name": "Brandenburger Tor, Brandenburger Straße, Historische Innenstadt, Innenstadt, Potsdam, Brandenburg, 14467, Germany",
+      "class": "tourism",
+      "type": "attraction",
+      "importance": 0.2940287400552381,
+      "address": {
+        "tourism": "Brandenburger Tor",
+        "road": "Brandenburger Straße",
+        "suburb": "Historische Innenstadt",
+        "city": "Potsdam",
+        "state": "Brandenburg",
+        "postcode": "14467",
+        "country": "Germany",
+        "country_code": "de"
+      },
+      "extratags": {
+        "image": "http://commons.wikimedia.org/wiki/File:Potsdam_brandenburger_tor.jpg",
+        "heritage": "4",
+        "wikidata": "Q695045",
+        "architect": "Carl von Gontard;Georg Christian Unger",
+        "wikipedia": "de:Brandenburger Tor (Potsdam)",
+        "wheelchair": "yes",
+        "description": "Kleines Brandenburger Tor in Potsdam",
+        "heritage:website": "http://www.bldam-brandenburg.de/images/stories/PDF/DML%202012/04-p-internet-13.pdf",
+        "heritage:operator": "bldam",
+        "architect:wikidata": "Q68768;Q95223",
+        "year_of_construction": "1771"
+      }
+   }
 ]
 ```

docs/api/Output.md

@@ -28,6 +28,7 @@ a single place (for reverse) of the following format:
       "city": "London",
       "state_district": "Greater London",
       "state": "England",
+      "ISO3166-2-lvl4": "GB-ENG",
       "postcode": "SW1A 2DU",
       "country": "United Kingdom",
       "country_code": "gb"
@@ -97,7 +98,10 @@ The GeocodeJSON format follows the
 The following feature attributes are implemented:
 
  * `osm_type`, `osm_id` - reference to the OSM object (unofficial extension, [see notes](#osm-reference))
- * `type` - value of the main tag of the object (e.g. residential, restaurant, ...)
+ * `type` - the 'address level' of the object ('house', 'street', `district`, `city`,
+            `county`, `state`, `country`, `locality`)
+ * `osm_key`- key of the main tag of the OSM object (e.g. boundary, highway, amenity)
+ * `osm_value` - value of the main tag of the OSM object (e.g. residential, restaurant)
  * `label` - full comma-separated address
  * `name` - localised name of the place
  * `housenumber`, `street`, `locality`, `district`, `postcode`, `city`,
@@ -126,6 +130,7 @@ formats depending on the API call.
   </result>
   <addressparts>
      <state>Bavaria</state>
+     <ISO3166-2-lvl4>DE-BY</ISO3166-2-lvl4>
      <country>Germany</country>
      <country_code>de</country_code>
   </addressparts>
@@ -179,6 +184,7 @@ Additional information requested with `addressdetails=1`, `extratags=1` and
     <city>London</city>
     <state_district>Greater London</state_district>
     <state>England</state>
+    <ISO3166-2-lvl4>GB-ENG</ISO3166-2-lvl4>
     <postcode>SW1A 2DU</postcode>
     <country>United Kingdom</country>
     <country_code>gb</country_code>
@@ -205,8 +211,8 @@ be more than one. The attributes of that element contain:
  * `ref` - content of `ref` tag if it exists
  * `lat`, `lon` - latitude and longitude of the centroid of the object
  * `boundingbox` - comma-separated list of corner coordinates ([see notes](#boundingbox))
- * `place_rank` - class [search rank](../develop/Ranking#search-rank)
- * `address_rank` - place [address rank](../develop/Ranking#address-rank)
+ * `place_rank` - class [search rank](../customize/Ranking.md#search-rank)
+ * `address_rank` - place [address rank](../customize/Ranking.md#address-rank)
  * `display_name` - full comma-separated address
  * `class`, `type` - key and value of the main OSM tag
  * `importance` - computed importance rank
@@ -230,7 +236,7 @@ on another server. It may even change its ID on the same server when it is
 removed and reimported while updating the database with fresh OSM data.
 It is thus not useful to treat it as permanent for later use.
 
-The combination `osm_type`+`osm_id` is slighly better but remember in
+The combination `osm_type`+`osm_id` is slightly better but remember in
 OpenStreetMap mappers can delete, split, recreate places (and those
 get a new `osm_id`), there is no link between those old and new ids.
 Places can also change their meaning without changing their `osm_id`,
@@ -279,12 +285,12 @@ with a designation label. Per default the following labels may appear:
 
  * continent
  * country, country_code
- * region, state, state_district, county
+ * region, state, state_district, county, ISO3166-2-lvl<admin_level>
  * municipality, city, town, village
  * city_district, district, borough, suburb, subdivision
  * hamlet, croft, isolated_dwelling
  * neighbourhood, allotments, quarter
- * city_block, residental, farm, farmyard, industrial, commercial, retail
+ * city_block, residential, farm, farmyard, industrial, commercial, retail
  * road
  * house_number, house_name
  * emergency, historic, military, natural, landuse, place, railway,

docs/api/Reverse.md

@@ -27,15 +27,15 @@ The main format of the reverse API is
 https://nominatim.openstreetmap.org/reverse?lat=<value>&lon=<value>&<params>
 ```
 
-where `lat` and `lon` are latitude and longitutde of a coordinate in WGS84
+where `lat` and `lon` are latitude and longitude of a coordinate in WGS84
 projection. The API returns exactly one result or an error when the coordinate
 is in an area with no OSM data coverage.
 
-Additional paramters are accepted as listed below.
+Additional parameters are accepted as listed below.
 
 !!! warning "Deprecation warning"
     The reverse API used to allow address lookup for a single OSM object by
-    its OSM id. This use is now deprecated. Use the [Address Lookup API](../Lookup)
+    its OSM id. This use is now deprecated. Use the [Address Lookup API](Lookup.md)
     instead.
 
 ### Output format
@@ -118,7 +118,7 @@ geometry. Topology is preserved in the result. (Default: 0.0)
 
 * `email=<valid email address>`
 
-If you are making large numbers of request please include an appropriate email
+If you are making a large number of requests, please include an appropriate email
 address to identify your requests. See Nominatim's [Usage Policy](https://operations.osmfoundation.org/policies/nominatim/) for more details.
 
 

docs/api/Search.md

@@ -51,6 +51,12 @@ Both query forms accept the additional parameters listed below.
 
 See [Place Output Formats](Output.md) for details on each format. (Default: jsonv2)
 
+!!! note
+    The Nominatim service at
+    [https://nominatim.openstreetmap.org](https://nominatim.openstreetmap.org)
+    has a different default behaviour for historical reasons. When the
+    `format` parameter is omitted, the request will be forwarded to the Web UI.
+
 * `json_callback=<string>`
 
 Wrap JSON output in a callback function ([JSONP](https://en.wikipedia.org/wiki/JSONP)) i.e. `<string>(<json>)`.

docs/api/Status.md

@@ -57,10 +57,11 @@ code and message, e.g.
 
 Possible status codes are
 
-   |     | message              | notes                                             |
-   |-----|----------------------|---------------------------------------------------|
-   | 700 | "No database"        | connection failed                                 |
-   | 701 | "Module failed"      | database could not load nominatim.so              |
-   | 702 | "Module call failed" | nominatim.so loaded but calling a function failed |
-   | 703 | "Query failed"       | test query against a database table failed        |
-   | 704 | "No value"           | test query worked but returned no results         |
+|     | message                        | notes                                                             |
+| --- | ------------------------------ | ----------------------------------------------------------------- |
+| 700 | "No database"                  | connection failed                                                 |
+| 701 | "Module failed"                | database could not load nominatim.so                              |
+| 702 | "Module call failed"           | nominatim.so loaded but calling a function failed                 |
+| 703 | "Query failed"                 | test query against a database table failed                        |
+| 704 | "No value"                     | test query worked but returned no results                         |
+| 705 | "Import date is not available" | No import dates were returned (enabling replication can fix this) |

docs/customize/Country-Settings.md

@@ -0,0 +1,149 @@
+# Customizing Per-Country Data
+
+Whenever an OSM is imported into Nominatim, the object is first assigned
+a country. Nominatim can use this information to adapt various aspects of
+the address computation to the local customs of the country. This section
+explains how country assignment works and the principal per-country
+localizations.
+
+## Country assignment
+
+Countries are assigned on the basis of country data from the OpenStreetMap
+input data itself. Countries are expected to be tagged according to the
+[administrative boundary schema](https://wiki.openstreetmap.org/wiki/Tag:boundary%3Dadministrative):
+a OSM relation with `boundary=administrative` and `admin_level=2`. Nominatim
+uses the country code to distinguish the countries.
+
+If there is no country data available for a point, then Nominatim uses the
+fallback data imported from `data/country_osm_grid.sql.gz`. This was computed
+from OSM data as well but is guaranteed to cover all countries.
+
+Some OSM objects may also be located outside any country, for example a buoy
+in the middle of the ocean. These object do not get any country assigned and
+get a default treatment when it comes to localized handling of data.
+
+## Per-country settings
+
+### Global country settings
+
+The main place to configure settings per country is the file
+`settings/country_settings.yaml`. This file has one section per country that
+is recognised by Nominatim. Each section is tagged with the country code
+(in lower case) and contains the different localization information. Only
+countries which are listed in this file are taken into account for computations.
+
+For example, the section for Andorra looks like this:
+
+```
+    partition: 35
+    languages: ca
+    names: !include country-names/ad.yaml
+    postcode:
+      pattern: "(ddd)"
+      output: AD\1
+```
+
+The individual settings are described below.
+
+#### `partition`
+
+Nominatim internally splits the data into multiple tables to improve
+performance. The partition number tells Nominatim into which table to put
+the country. This is purely internal management and has no effect on the
+output data.
+
+The default is to have one partition per country.
+
+#### `languages`
+
+A comma-separated list of ISO-639 language codes of default languages in the
+country. These are the languages used in name tags without a language suffix.
+Note that this is not necessarily the same as the list of official languages
+in the country. There may be officially recognised languages in a country
+which are only ever used in name tags with the appropriate language suffixes.
+Conversely, a non-official language may appear a lot in the name tags, for
+example when used as an unofficial Lingua Franca.
+
+List the languages in order of frequency of appearance with the most frequently
+used language first. It is not recommended to add languages when there are only
+very few occurrences.
+
+If only one language is listed, then Nominatim will 'auto-complete' the
+language of names without an explicit language-suffix.
+
+#### `names`
+
+List of names of the country and its translations. These names are used as
+a baseline. It is always possible to search countries by the given names, no
+matter what other names are in the OSM data. They are also used as a fallback
+when a needed translation is not available.
+
+!!! Note
+    The list of names per country is currently fairly large because Nominatim
+    supports translations in many languages per default. That is why the
+    name lists have been separated out into extra files. You can find the
+    name lists in the file `settings/country-names/<country code>.yaml`.
+    The names section in the main country settings file only refers to these
+    files via the special `!include` directive.
+
+#### `postcode`
+
+Describes the format of the postcode that is in use in the country.
+
+When a country has no official postcodes, set this to no. Example:
+
+```
+ae:
+    postcode: no
+```
+
+When a country has a postcode, you need to state the postcode pattern and
+the default output format. Example:
+
+```
+bm:
+    postcode:
+      pattern: "(ll)[ -]?(dd)"
+      output: \1 \2
+```
+
+The **pattern** is a regular expression that describes the possible formats
+accepted as a postcode. The pattern follows the standard syntax for
+[regular expressions in Python](https://docs.python.org/3/library/re.html#regular-expression-syntax)
+with two extra shortcuts: `d` is a shortcut for a single digit([0-9])
+and `l` for a single ASCII letter ([A-Z]).
+
+Use match groups to indicate groups in the postcode that may optionally be
+separated with a space or a hyphen.
+
+For example, the postcode for Bermuda above always consists of two letters
+and two digits. They may optionally be separated by a space or hyphen. That
+means that Nominatim will consider `AB56`, `AB 56` and `AB-56` spelling variants
+for one and the same postcode.
+
+Never add the country code in front of the postcode pattern. Nominatim will
+automatically accept variants with a country code prefix for all postcodes.
+
+The **output** field is an optional field that describes what the canonical
+spelling of the postcode should be. The format is the
+[regular expression expand syntax](https://docs.python.org/3/library/re.html#re.Match.expand) referring back to the bracket groups in the pattern.
+
+Most simple postcodes only have one spelling variant. In that case, the
+**output** can be omitted. The postcode will simply be used as is.
+
+In the Bermuda example above, the canonical spelling would be to have a space
+between letters and digits.
+
+!!! Warning
+    When your postcode pattern covers multiple variants of the postcode, then
+    you must explicitly state the canonical output or Nominatim will not
+    handle the variations correctly.
+
+### Other country-specific configuration
+
+There are some other configuration files where you can set localized settings
+according to the assigned country. These are:
+
+ * [Place ranking configuration](Ranking.md)
+
+Please see the linked documentation sections for more information.

docs/customize/Import-Styles.md

@@ -10,7 +10,7 @@ option. There are a number of default styles, which are explained in detail
 in the [Import section](../admin/Import.md#filtering-imported-data). These
 standard styles may be referenced by their name.
 
-You can also create your own custom syle. Put the style file into your
+You can also create your own custom style. Put the style file into your
 project directory and then set `NOMINATIM_IMPORT_STYLE` to the name of the file.
 It is always recommended to start with one of the standard styles and customize
 those. You find the standard styles under the name `import-<stylename>.style`

docs/customize/Importance.md

@@ -0,0 +1,49 @@
+## Importance
+
+Search requests can yield multiple results which match equally well with
+the original query. In such case Nominatim needs to order the results
+according to a different criterion: importance. This is a measure for how
+likely it is that a user will search for a given place. This section explains
+the sources Nominatim uses for computing importance of a place and how to
+customize them.
+
+### How importance is computed
+
+The main value for importance is derived from page ranking values for Wikipedia
+pages for a place. For places that do not have their own
+Wikipedia page, a formula is used that derives a static importance from the
+places [search rank](../customize/Ranking.md#search-rank).
+
+In a second step, a secondary importance value is added which is meant to
+represent how well-known the general area is where the place is located. It
+functions as a tie-breaker between places with very similar primary
+importance values.
+
+nominatim.org has preprocessed importance tables for the
+[primary Wikipedia rankings](https://nominatim.org/data/wikimedia-importance.sql.gz)
+and for a secondary importance based on the number of tile views on openstreetmap.org.
+
+### Customizing secondary importance
+
+The secondary importance is implemented as a simple
+[Postgis raster](https://postgis.net/docs/raster.html) table, where Nominatim
+looks up the value for the coordinates of the centroid of a place. You can
+provide your own secondary importance raster in form of an SQL file named
+`secondary_importance.sql.gz` in your project directory.
+
+The SQL file needs to drop and (re)create a table `secondary_importance` which
+must as a minimum contain a column `rast` of type `raster`. The raster must
+be in EPSG:4326 and contain 16bit unsigned ints
+(`raster_constraint_pixel_types(rast) = '{16BUI}'). Any other columns in the
+table will be ignored. You must furthermore create an index as follows:
+
+```
+CREATE INDEX ON secondary_importance USING gist(ST_ConvexHull(gist))
+```
+
+The following raster2pgsql command will create a table that conforms to
+the requirements:
+
+```
+raster2pgsql -I -C -Y -d -t 128x128 input.tiff public.secondary_importance
+```

docs/customize/Settings.md

@@ -148,6 +148,29 @@ Setting this option to 'yes' means that Nominatim skips reindexing of contained
 objects when the area becomes too large.
 
 
+#### NOMINATIM_UPDATE_FORWARD_DEPENDENCIES
+
+| Summary            |                                                     |
+| --------------     | --------------------------------------------------- |
+| **Description:**   | Forward geometry changes to dependet objects |
+| **Format:**        | bool |
+| **Default:**       | no |
+| **Comment:**       | EXPERT ONLY. Must not be enabled after import. |
+
+The geometry of OSM ways and relations may change when a node that is part
+of the object is moved around. These changes are not propagated per default.
+The geometry of ways/relations is only updated the next time that the object
+itself is touched. When enabling this option, then dependent objects will
+be marked for update when one of its member objects changes.
+
+Enabling this option may slow down updates significantly.
+
+!!! warning
+    If you want to enable this option, it must be set already on import.
+    Do not enable this option on an existing database that was imported with
+    NOMINATIM_UPDATE_FORWARD_DEPENDENCIES=no.
+    Updates will become unusably slow.
+
 #### NOMINATIM_LANGUAGES
 
 | Summary            |                                                     |
@@ -189,7 +212,7 @@ will be used.
 | **Description:**   | Enable searching for Tiger house number data |
 | **Format:**        | boolean |
 | **Default:**       | no |
-| **After Changes:** | run `nominatim --refresh --functions` |
+| **After Changes:** | run `nominatim refresh --functions` |
 
 When this setting is enabled, search and reverse queries also take data
 from [Tiger house number data](Tiger.md) into account.
@@ -202,7 +225,7 @@ from [Tiger house number data](Tiger.md) into account.
 | **Description:**   | Enable searching in external house number tables |
 | **Format:**        | boolean |
 | **Default:**       | no |
-| **After Changes:** | run `nominatim --refresh --functions` |
+| **After Changes:** | run `nominatim refresh --functions` |
 | **Comment:**       | Do not use. |
 
 When this setting is enabled, search queries also take data from external
@@ -643,7 +666,7 @@ The entries in the log file have the following format:
     <request time> <execution time in s> <number of results> <type> "<query string>"
 
 Request time is the time when the request was started. The execution time is
-given in ms and corresponds to the time the query took executing in PHP.
+given in seconds and corresponds to the time the query took executing in PHP.
 type contains the name of the endpoint used.
 
 Can be used as the same time as NOMINATIM_LOG_DB.

docs/customize/Tiger.md

@@ -5,22 +5,22 @@ address set to complement the OSM house number data in the US. You can add
 TIGER data to your own Nominatim instance by following these steps. The
 entire US adds about 10GB to your database.
 
-  1. Get preprocessed TIGER 2021 data:
+  1. Get preprocessed TIGER data:
 
         cd $PROJECT_DIR
-        wget https://nominatim.org/data/tiger2021-nominatim-preprocessed.csv.tar.gz
+        wget https://nominatim.org/data/tiger-nominatim-preprocessed-latest.csv.tar.gz
 
   2. Import the data into your Nominatim database:
 
-        nominatim add-data --tiger-data tiger2021-nominatim-preprocessed.csv.tar.gz
+        nominatim add-data --tiger-data tiger-nominatim-preprocessed-latest.csv.tar.gz
 
-  3. Enable use of the Tiger data in your `.env` by adding:
+  3. Enable use of the Tiger data in your existing `.env` file by adding:
 
         echo NOMINATIM_USE_US_TIGER_DATA=yes >> .env
 
   4. Apply the new settings:
 
-        nominatim refresh --functions
+        nominatim refresh --functions --website
 
 
 See the [TIGER-data project](https://github.com/osm-search/TIGER-data) for more

docs/customize/Tokenizers.md

@@ -19,7 +19,22 @@ they can be configured.
 
 The legacy tokenizer implements the analysis algorithms of older Nominatim
 versions. It uses a special Postgresql module to normalize names and queries.
-This tokenizer is currently the default.
+This tokenizer is automatically installed and used when upgrading an older
+database. It should not be used for new installations anymore.
+
+### Compiling the PostgreSQL module
+
+The tokeinzer needs a special C module for PostgreSQL which is not compiled
+by default. If you need the legacy tokenizer, compile Nominatim as follows:
+
+```
+mkdir build
+cd build
+cmake -DBUILD_MODULE=on
+make
+```
+
+### Enabling the tokenizer
 
 To enable the tokenizer add the following line to your project configuration:
 
@@ -47,6 +62,7 @@ normalization functions are hard-coded.
 The ICU tokenizer uses the [ICU library](http://site.icu-project.org/) to
 normalize names and queries. It also offers configurable decomposition and
 abbreviation handling.
+This tokenizer is currently the default.
 
 To enable the tokenizer add the following line to your project configuration:
 
@@ -99,6 +115,9 @@ token-analysis:
           - words:
               - road -> rd
               - bridge -> bdge,br,brdg,bri,brg
+      mutations:
+          - pattern: 'ä'
+            replacements: ['ä', 'ae']
 ```
 
 The configuration file contains four sections:
@@ -178,6 +197,30 @@ The following is a list of sanitizers that are shipped with Nominatim.
     rendering:
         heading_level: 6
 
+##### clean-housenumbers
+
+::: nominatim.tokenizer.sanitizers.clean_housenumbers
+    selection:
+        members: False
+    rendering:
+        heading_level: 6
+
+##### clean-postcodes
+
+::: nominatim.tokenizer.sanitizers.clean_postcodes
+    selection:
+        members: False
+    rendering:
+        heading_level: 6
+
+##### clean-tiger-tags
+
+::: nominatim.tokenizer.sanitizers.clean_tiger_tags
+    selection:
+        members: False
+    rendering:
+        heading_level: 6
+
 
 
 #### Token Analysis
@@ -196,21 +239,25 @@ by a sanitizer (see for example the
 The token-analysis section contains the list of configured analyzers. Each
 analyzer must have an `id` parameter that uniquely identifies the analyzer.
 The only exception is the default analyzer that is used when no special
-analyzer was selected.
+analyzer was selected. There are analysers with special ids:
+
+ * '@housenumber'. If an analyzer with that name is present, it is used
+   for normalization of house numbers.
+ * '@potcode'. If an analyzer with that name is present, it is used
+   for normalization of postcodes.
 
 Different analyzer implementations may exist. To select the implementation,
-the `analyzer` parameter must be set. Currently there is only one implementation
-`generic` which is described in the following.
+the `analyzer` parameter must be set. The different implementations are
+described in the following.
 
 ##### Generic token analyzer
 
-The generic analyzer is able to create variants from a list of given
-abbreviation and decomposition replacements. It takes one optional parameter
-`variants` which lists the replacements to apply. If the section is
-omitted, then the generic analyzer becomes a simple analyzer that only
-applies the transliteration.
+The generic analyzer `generic` is able to create variants from a list of given
+abbreviation and decomposition replacements and introduce spelling variations.
 
-The variants section defines lists of replacements which create alternative
+###### Variants
+
+The optional 'variants' section defines lists of replacements which create alternative
 spellings of a name. To create the variants, a name is scanned from left to
 right and the longest matching replacement is applied until the end of the
 string is reached.
@@ -296,6 +343,48 @@ decomposition has an effect here on the source as well. So a rule
 means that for a word like `hauptstrasse` four variants are created:
 `hauptstrasse`, `haupt strasse`, `hauptstr` and `haupt str`.
 
+###### Mutations
+
+The 'mutation' section in the configuration describes an additional set of
+replacements to be applied after the variants have been computed.
+
+Each mutation is described by two parameters: `pattern` and `replacements`.
+The pattern must contain a single regular expression to search for in the
+variant name. The regular expressions need to follow the syntax for
+[Python regular expressions](file:///usr/share/doc/python3-doc/html/library/re.html#regular-expression-syntax).
+Capturing groups are not permitted.
+`replacements` must contain a list of strings that the pattern
+should be replaced with. Each occurrence of the pattern is replaced with
+all given replacements. Be mindful of combinatorial explosion of variants.
+
+###### Modes
+
+The generic analyser supports a special mode `variant-only`. When configured
+then it consumes the input token and emits only variants (if any exist). Enable
+the mode by adding:
+
+```
+  mode: variant-only
+```
+
+to the analyser configuration.
+
+##### Housenumber token analyzer
+
+The analyzer `housenumbers` is purpose-made to analyze house numbers. It
+creates variants with optional spaces between numbers and letters. Thus,
+house numbers of the form '3 a', '3A', '3-A' etc. are all considered equivalent.
+
+The analyzer cannot be customized.
+
+##### Postcode token analyzer
+
+The analyzer `postcodes` is pupose-made to analyze postcodes. It supports
+a 'lookup' varaint of the token, which produces variants with optional
+spaces. Use together with the clean-postcodes sanitizer.
+
+The analyzer cannot be customized.
+
 ### Reconfiguration
 
 Changing the configuration after the import is currently not possible, although

docs/develop/Database-Layout.md

@@ -119,7 +119,7 @@ to compute the address relations between places. These tables are partitioned.
 Each country is assigned a partition number in the country_name table (see
 below) and the data is then split between a set of tables, one for each
 partition. Note that Nominatim still manually manages partitioned tables.
-Native support for partitions in PostgreSQL only became useable with version 13.
+Native support for partitions in PostgreSQL only became usable with version 13.
 It will be a little while before Nominatim drops support for older versions.
 
 ![address tables](address-tables.svg)
@@ -155,9 +155,9 @@ Nominatim also creates a number of static tables at import:
    default languages and saves the assignment of countries to partitions.
  * `country_osm_grid` provides a fallback for country geometries
 
-## Auxilary data tables
+## Auxiliary data tables
 
-Finally there are some table for auxillary data:
+Finally there are some table for auxiliary data:
 
  * `location_property_tiger` - saves housenumber from the Tiger import. Its
    layout is similar to that of `location_propoerty_osmline`.

docs/develop/Development-Environment.md

@@ -1,6 +1,6 @@
 # Setting up Nominatim for Development
 
-This chapter gives an overview how to set up Nominatim for developement
+This chapter gives an overview how to set up Nominatim for development
 and how to run tests.
 
 !!! Important
@@ -30,15 +30,18 @@ unit tests (using PHPUnit for PHP code and pytest for Python code).
 It has the following additional requirements:
 
 * [behave test framework](https://behave.readthedocs.io) >= 1.2.6
-* [phpunit](https://phpunit.de) >= 7.3
+* [phpunit](https://phpunit.de) (9.5 is known to work)
 * [PHP CodeSniffer](https://github.com/squizlabs/PHP_CodeSniffer)
-* [Pylint](https://pylint.org/) (2.6.0 is used for the CI)
+* [Pylint](https://pylint.org/) (CI always runs the latest version from pip)
+* [mypy](http://mypy-lang.org/) (plus typing information for external libs)
+* [Python Typing Extensions](https://github.com/python/typing_extensions) (for Python < 3.9)
 * [pytest](https://pytest.org)
 
 The documentation is built with mkdocs:
 
 * [mkdocs](https://www.mkdocs.org/) >= 1.1.2
-* [mkdocstrings](https://mkdocstrings.github.io/)
+* [mkdocstrings](https://mkdocstrings.github.io/) >= 0.16
+* [mkdocstrings-python-legacy](https://mkdocstrings.github.io/python-legacy/)
 
 ### Installing prerequisites on Ubuntu/Debian
 
@@ -50,9 +53,10 @@ To install all necessary packages run:
 
 ```sh
 sudo apt install php-cgi phpunit php-codesniffer \
-                 python3-pip python3-setuptools python3-dev pylint
+                 python3-pip python3-setuptools python3-dev
 
-pip3 install --user behave mkdocs mkdocstrings pytest
+pip3 install --user behave mkdocs mkdocstrings pytest pylint \
+                    mypy types-PyYAML types-jinja2 types-psycopg2 types-psutil
 ```
 
 The `mkdocs` executable will be located in `.local/bin`. You may have to add

docs/develop/ICU-Tokenizer-Modules.md

@@ -0,0 +1,227 @@
+# Writing custom sanitizer and token analysis modules for the ICU tokenizer
+
+The [ICU tokenizer](../customize/Tokenizers.md#icu-tokenizer) provides a
+highly customizable method to pre-process and normalize the name information
+of the input data before it is added to the search index. It comes with a
+selection of sanitizers and token analyzers which you can use to adapt your
+installation to your needs. If the provided modules are not enough, you can
+also provide your own implementations. This section describes the API
+of sanitizers and token analysis.
+
+!!! warning
+    This API is currently in early alpha status. While this API is meant to
+    be a public API on which other sanitizers and token analyzers may be
+    implemented, it is not guaranteed to be stable at the moment.
+
+
+## Using non-standard sanitizers and token analyzers
+
+Sanitizer names (in the `step` property) and token analysis names (in the
+`analyzer`) may refer to externally supplied modules. There are two ways
+to include external modules: through a library or from the project directory.
+
+To include a module from a library, use the absolute import path as name and
+make sure the library can be found in your PYTHONPATH.
+
+To use a custom module without creating a library, you can put the module
+somewhere in your project directory and then use the relative path to the
+file. Include the whole name of the file including the `.py` ending.
+
+## Custom sanitizer modules
+
+A sanitizer module must export a single factory function `create` with the
+following signature:
+
+``` python
+def create(config: SanitizerConfig) -> Callable[[ProcessInfo], None]
+```
+
+The function receives the custom configuration for the sanitizer and must
+return a callable (function or class) that transforms the name and address
+terms of a place. When a place is processed, then a `ProcessInfo` object
+is created from the information that was queried from the database. This
+object is sequentially handed to each configured sanitizer, so that each
+sanitizer receives the result of processing from the previous sanitizer.
+After the last sanitizer is finished, the resulting name and address lists
+are forwarded to the token analysis module.
+
+Sanitizer functions are instantiated once and then called for each place
+that is imported or updated. They don't need to be thread-safe.
+If multi-threading is used, each thread creates their own instance of
+the function.
+
+### Sanitizer configuration
+
+::: nominatim.tokenizer.sanitizers.config.SanitizerConfig
+    rendering:
+        show_source: no
+        heading_level: 6
+
+### The main filter function of the sanitizer
+
+The filter function receives a single object of type `ProcessInfo`
+which has with three members:
+
+ * `place`: read-only information about the place being processed.
+   See PlaceInfo below.
+ * `names`: The current list of names for the place. Each name is a
+   PlaceName object.
+ * `address`: The current list of address names for the place. Each name
+   is a PlaceName object.
+
+While the `place` member is provided for information only, the `names` and
+`address` lists are meant to be manipulated by the sanitizer. It may add and
+remove entries, change information within a single entry (for example by
+adding extra attributes) or completely replace the list with a different one.
+
+#### PlaceInfo - information about the place
+
+::: nominatim.data.place_info.PlaceInfo
+    rendering:
+        show_source: no
+        heading_level: 6
+
+
+#### PlaceName - extended naming information
+
+::: nominatim.data.place_name.PlaceName
+    rendering:
+        show_source: no
+        heading_level: 6
+
+
+### Example: Filter for US street prefixes
+
+The following sanitizer removes the directional prefixes from street names
+in the US:
+
+``` python
+import re
+
+def _filter_function(obj):
+    if obj.place.country_code == 'us' \
+       and obj.place.rank_address >= 26 and obj.place.rank_address <= 27:
+        for name in obj.names:
+            name.name = re.sub(r'^(north|south|west|east) ',
+                               '',
+                               name.name,
+                               flags=re.IGNORECASE)
+
+def create(config):
+    return _filter_function
+```
+
+This is the most simple form of a sanitizer module. If defines a single
+filter function and implements the required `create()` function by returning
+the filter.
+
+The filter function first checks if the object is interesting for the
+sanitizer. Namely it checks if the place is in the US (through `country_code`)
+and it the place is a street (a `rank_address` of 26 or 27). If the
+conditions are met, then it goes through all available names and
+removes any leading directional prefix using a simple regular expression.
+
+Save the source code in a file in your project directory, for example as
+`us_streets.py`. Then you can use the sanitizer in your `icu_tokenizer.yaml`:
+
+``` yaml
+...
+sanitizers:
+    - step: us_streets.py
+...
+```
+
+!!! warning
+    This example is just a simplified show case on how to create a sanitizer.
+    It is not really read for real-world use: while the sanitizer would
+    correcly transform `West 5th Street` into `5th Street`. it would also
+    shorten a simple `North Street` to `Street`.
+
+For more sanitizer examples, have a look at the sanitizers provided by Nominatim.
+They can be found in the directory
+[`nominatim/tokenizer/sanitizers`](https://github.com/osm-search/Nominatim/tree/master/nominatim/tokenizer/sanitizers).
+
+
+## Custom token analysis module
+
+::: nominatim.tokenizer.token_analysis.base.AnalysisModule
+    rendering:
+        show_source: no
+        heading_level: 6
+
+
+::: nominatim.tokenizer.token_analysis.base.Analyzer
+    rendering:
+        show_source: no
+        heading_level: 6
+
+### Example: Creating acronym variants for long names
+
+The following example of a token analysis module creates acronyms from
+very long names and adds them as a variant:
+
+``` python
+class AcronymMaker:
+    """ This class is the actual analyzer.
+    """
+    def __init__(self, norm, trans):
+        self.norm = norm
+        self.trans = trans
+
+
+    def get_canonical_id(self, name):
+        # In simple cases, the normalized name can be used as a canonical id.
+        return self.norm.transliterate(name.name).strip()
+
+
+    def compute_variants(self, name):
+        # The transliterated form of the name always makes up a variant.
+        variants = [self.trans.transliterate(name)]
+
+        # Only create acronyms from very long words.
+        if len(name) > 20:
+            # Take the first letter from each word to form the acronym.
+            acronym = ''.join(w[0] for w in name.split())
+            # If that leds to an acronym with at least three letters,
+            # add the resulting acronym as a variant.
+            if len(acronym) > 2:
+                # Never forget to transliterate the variants before returning them.
+                variants.append(self.trans.transliterate(acronym))
+
+        return variants
+
+# The following two functions are the module interface.
+
+def configure(rules, normalizer, transliterator):
+    # There is no configuration to parse and no data to set up.
+    # Just return an empty configuration.
+    return None
+
+
+def create(normalizer, transliterator, config):
+    # Return a new instance of our token analysis class above.
+    return AcronymMaker(normalizer, transliterator)
+```
+
+Given the name `Trans-Siberian Railway`, the code above would return the full
+name `Trans-Siberian Railway` and the acronym `TSR` as variant, so that
+searching would work for both.
+
+## Sanitizers vs. Token analysis - what to use for variants?
+
+It is not always clear when to implement variations in the sanitizer and
+when to write a token analysis module. Just take the acronym example
+above: it would also have been possible to write a sanitizer which adds the
+acronym as an additional name to the name list. The result would have been
+similar. So which should be used when?
+
+The most important thing to keep in mind is that variants created by the
+token analysis are only saved in the word lookup table. They do not need
+extra space in the search index. If there are many spelling variations, this
+can mean quite a significant amount of space is saved.
+
+When creating additional names with a sanitizer, these names are completely
+independent. In particular, they can be fed into different token analysis
+modules. This gives a much greater flexibility but at the price that the
+additional names increase the size of the search index.
+

docs/develop/Indexing.md

@@ -78,7 +78,7 @@ The inheritance is computed in the data preparation step.
 The prepared place information is handed to the tokenizer next. This is a
 Python module responsible for processing the names  from both name and address
 terms and building up the word index from them. The process is explained in
-more detail in the [Tokenizer chapter](Tokenizer.md).
+more detail in the [Tokenizer chapter](Tokenizers.md).
 
 ### Address processing
 

docs/develop/Testing.md

@@ -22,8 +22,8 @@ This test directory is sturctured as follows:
   |
   +-   php         PHP unit tests
   +-   python      Python unit tests
-  +-   scenes      Geometry test data
   +-   testdb      Base data for generating API test database
+  +-   testdata    Additional test data used by unit tests
 ```
 
 ## PHP Unit Tests (`test/php`)

docs/develop/Tokenizers.md

@@ -93,7 +93,7 @@ for a custom tokenizer implementation.
 
 Nominatim expects two files for a tokenizer:
 
-* `nominiatim/tokenizer/<NAME>_tokenizer.py` containing the Python part of the
+* `nominatim/tokenizer/<NAME>_tokenizer.py` containing the Python part of the
   implementation
 * `lib-php/tokenizer/<NAME>_tokenizer.php` with the PHP part of the
   implementation
@@ -105,7 +105,7 @@ functions. By convention, these should be placed in `lib-sql/tokenizer`.
 If the tokenizer has a default configuration file, this should be saved in
 the `settings/<NAME>_tokenizer.<SUFFIX>`.
 
-### Configuration and Persistance
+### Configuration and Persistence
 
 Tokenizers may define custom settings for their configuration. All settings
 must be prefixed with `NOMINATIM_TOKENIZER_`. Settings may be transient or
@@ -245,11 +245,11 @@ Currently, tokenizers are encouraged to make sure that matching works against
 both the search token list and the match token list.
 
 ```sql
-FUNCTION token_normalized_postcode(postcode TEXT) RETURNS TEXT
+FUNCTION token_get_postcode(info JSONB) RETURNS TEXT
 ```
 
-Return the normalized version of the given postcode. This function must return
-the same value as the Python function `AbstractAnalyzer->normalize_postcode()`.
+Return the postcode for the object, if any exists. The postcode must be in
+the form that should also be presented to the end-user.
 
 ```sql
 FUNCTION token_strip_info(info JSONB) RETURNS JSONB

docs/develop/data-sources.md

@@ -13,7 +13,7 @@ More details in [osm-search/country-grid-data](https://github.com/osm-search/cou
 
 ## US Census TIGER
 
-For the United States you can choose to import additonal street-level data.
+For the United States you can choose to import additional street-level data.
 The data isn't mixed into OSM data but queried as fallback when no OSM
 result can be found.
 

docs/extra.css

@@ -14,10 +14,11 @@ th {
     background-color: #eee;
 }
 
-/* Indentation for mkdocstrings.
-div.doc-contents:not(.first) {
-  padding-left: 25px;
-  border-left: 4px solid rgba(230, 230, 230);
-  margin-bottom: 60px;
-}*/
+.doc-object h6 {
+    margin-bottom: 0.8em;
+    font-size: 120%;
+}
 
+.doc-object {
+    margin-bottom: 1.3em;
+}

docs/mkdocs.yml

@@ -3,7 +3,7 @@ theme: readthedocs
 docs_dir: ${CMAKE_CURRENT_BINARY_DIR}
 site_url: https://nominatim.org
 repo_url: https://github.com/openstreetmap/Nominatim
-pages:
+nav:
     - 'Introduction' : 'index.md'
     - 'API Reference':
         - 'Overview': 'api/Overview.md'
@@ -28,7 +28,9 @@ pages:
         - 'Overview': 'customize/Overview.md'
         - 'Import Styles': 'customize/Import-Styles.md'
         - 'Configuration Settings': 'customize/Settings.md'
+        - 'Per-Country Data': 'customize/Country-Settings.md'
         - 'Place Ranking' : 'customize/Ranking.md'
+        - 'Importance' : 'customize/Importance.md'
         - 'Tokenizers' : 'customize/Tokenizers.md'
         - 'Special Phrases': 'customize/Special-Phrases.md'
         - 'External data: US housenumbers from TIGER': 'customize/Tiger.md'
@@ -38,14 +40,14 @@ pages:
         - 'Database Layout' : 'develop/Database-Layout.md'
         - 'Indexing' : 'develop/Indexing.md'
         - 'Tokenizers' : 'develop/Tokenizers.md'
+        - 'Custom modules for ICU tokenizer': 'develop/ICU-Tokenizer-Modules.md'
         - 'Setup for Development' : 'develop/Development-Environment.md'
         - 'Testing' : 'develop/Testing.md'
         - 'External Data Sources': 'develop/data-sources.md'
     - 'Appendix':
-        - 'Installation on CentOS 7' : 'appendix/Install-on-Centos-7.md'
-        - 'Installation on CentOS 8' : 'appendix/Install-on-Centos-8.md'
         - 'Installation on Ubuntu 18' : 'appendix/Install-on-Ubuntu-18.md'
         - 'Installation on Ubuntu 20' : 'appendix/Install-on-Ubuntu-20.md'
+        - 'Installation on Ubuntu 22' : 'appendix/Install-on-Ubuntu-22.md'
 markdown_extensions:
     - codehilite
     - admonition
@@ -57,7 +59,7 @@ plugins:
     - search
     - mkdocstrings:
         handlers:
-          python:
+          python-legacy:
             rendering:
               show_source: false
               show_signature_annotations: false

lib-php/AddressDetails.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -84,6 +92,10 @@ class AddressDetails
                     || $aLine['class'] == 'place')
             ) {
                 $aAddress[$sTypeLabel] = $sName;
+
+                if (!empty($aLine['name'])) {
+                    $this->addSubdivisionCode($aAddress, $aLine['admin_level'], $aLine['name']);
+                }
             }
         }
 
@@ -166,4 +178,14 @@ class AddressDetails
     {
         return $this->aAddressLines;
     }
+
+    private function addSubdivisionCode(&$aAddress, $iAdminLevel, $nameDetails)
+    {
+        if (is_string($nameDetails)) {
+            $nameDetails = json_decode('{' . str_replace('"=>"', '":"', $nameDetails) . '}', true);
+        }
+        if (!empty($nameDetails['ISO3166-2'])) {
+            $aAddress["ISO3166-2-lvl$iAdminLevel"] = $nameDetails['ISO3166-2'];
+        }
+    }
 }

lib-php/ClassTypes.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim\ClassTypes;
 

lib-php/DB.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -38,6 +46,9 @@ class DB
 
         $conn->exec("SET DateStyle TO 'sql,european'");
         $conn->exec("SET client_encoding TO 'utf-8'");
+        // Disable JIT and parallel workers. They interfere badly with search SQL.
+        $conn->exec("UPDATE pg_settings SET setting = -1 WHERE name = 'jit_above_cost'");
+        $conn->exec("UPDATE pg_settings SET setting = 0 WHERE name = 'max_parallel_workers_per_gather'");
         $iMaxExecution = ini_get('max_execution_time');
         if ($iMaxExecution > 0) {
             $conn->setAttribute(\PDO::ATTR_TIMEOUT, $iMaxExecution); // seconds

lib-php/DatabaseError.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 

lib-php/DebugHtml.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -127,7 +135,7 @@ class Debug
 
     public static function printSQL($sSQL)
     {
-        echo '<p><tt><font color="#aaa">'.$sSQL.'</font></tt></p>'."\n";
+        echo '<p><tt><font color="#aaa">'.htmlspecialchars($sSQL, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401).'</font></tt></p>'."\n";
     }
 
     private static function outputVar($mVar, $sPreNL)
@@ -170,11 +178,12 @@ class Debug
         }
 
         if (is_string($mVar)) {
-            echo "'$mVar'";
-            return strlen($mVar) + 2;
+            $sOut = "'$mVar'";
+        } else {
+            $sOut = (string)$mVar;
         }
 
-        echo (string)$mVar;
-        return strlen((string)$mVar);
+        echo htmlspecialchars($sOut, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401);
+        return strlen($sOut);
     }
 }

lib-php/DebugNone.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 

lib-php/Geocode.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -95,7 +103,7 @@ class Geocode
         }
 
         $this->iFinalLimit = $iLimit;
-        $this->iLimit = $iLimit + min($iLimit, 10);
+        $this->iLimit = $iLimit + max($iLimit, 10);
     }
 
     public function setFeatureType($sFeatureType)
@@ -182,7 +190,7 @@ class Geocode
 
         $this->bFallback = $oParams->getBool('fallback', $this->bFallback);
 
-        // List of excluded Place IDs - used for more acurate pageing
+        // List of excluded Place IDs - used for more accurate pageing
         $sExcluded = $oParams->getStringList('exclude_place_ids');
         if ($sExcluded) {
             foreach ($sExcluded as $iExcludedPlaceID) {
@@ -609,16 +617,15 @@ class Geocode
                     }
                     $aReverseGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $oValidTokens);
 
-                    foreach ($aGroupedSearches as $aSearches) {
+                    foreach ($aReverseGroupedSearches as $aSearches) {
                         foreach ($aSearches as $aSearch) {
-                            if (!isset($aReverseGroupedSearches[$aSearch->getRank()])) {
-                                $aReverseGroupedSearches[$aSearch->getRank()] = array();
+                            if (!isset($aGroupedSearches[$aSearch->getRank()])) {
+                                $aGroupedSearches[$aSearch->getRank()] = array();
                             }
-                            $aReverseGroupedSearches[$aSearch->getRank()][] = $aSearch;
+                            $aGroupedSearches[$aSearch->getRank()][] = $aSearch;
                         }
                     }
 
-                    $aGroupedSearches = $aReverseGroupedSearches;
                     ksort($aGroupedSearches);
                 }
             } else {
@@ -836,7 +843,9 @@ class Geocode
                 $aResult['importance'] = 0.001;
                 $aResult['foundorder'] = $aResult['addressimportance'];
             } else {
-                $aResult['importance'] = max(0.001, $aResult['importance']);
+                if ($aResult['importance'] == 0) {
+                    $aResult['importance'] = 0.0001;
+                }
                 $aResult['importance'] *= $this->viewboxImportanceFactor(
                     $aResult['lon'],
                     $aResult['lat']
@@ -865,7 +874,7 @@ class Geocode
                 $iCountWords = 0;
                 $sAddress = $aResult['langaddress'];
                 foreach ($aRecheckWords as $i => $sWord) {
-                    if (stripos($sAddress, $sWord)!==false) {
+                    if (grapheme_stripos($sAddress, $sWord)!==false) {
                         $iCountWords++;
                         if (preg_match('/(^|,)\s*'.preg_quote($sWord, '/').'\s*(,|$)/', $sAddress)) {
                             $iCountWords += 0.1;

lib-php/ParameterParser.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -14,7 +22,10 @@ class ParameterParser
 
     public function getBool($sName, $bDefault = false)
     {
-        if (!isset($this->aParams[$sName]) || strlen($this->aParams[$sName]) == 0) {
+        if (!isset($this->aParams[$sName])
+            || !is_string($this->aParams[$sName])
+            || strlen($this->aParams[$sName]) == 0
+        ) {
             return $bDefault;
         }
 
@@ -23,7 +34,7 @@ class ParameterParser
 
     public function getInt($sName, $bDefault = false)
     {
-        if (!isset($this->aParams[$sName])) {
+        if (!isset($this->aParams[$sName]) || is_array($this->aParams[$sName])) {
             return $bDefault;
         }
 
@@ -36,7 +47,7 @@ class ParameterParser
 
     public function getFloat($sName, $bDefault = false)
     {
-        if (!isset($this->aParams[$sName])) {
+        if (!isset($this->aParams[$sName]) || is_array($this->aParams[$sName])) {
             return $bDefault;
         }
 
@@ -49,7 +60,10 @@ class ParameterParser
 
     public function getString($sName, $bDefault = false)
     {
-        if (!isset($this->aParams[$sName]) || strlen($this->aParams[$sName]) == 0) {
+        if (!isset($this->aParams[$sName])
+            || !is_string($this->aParams[$sName])
+            || strlen($this->aParams[$sName]) == 0
+        ) {
             return $bDefault;
         }
 
@@ -58,11 +72,14 @@ class ParameterParser
 
     public function getSet($sName, $aValues, $sDefault = false)
     {
-        if (!isset($this->aParams[$sName]) || strlen($this->aParams[$sName]) == 0) {
+        if (!isset($this->aParams[$sName])
+            || !is_string($this->aParams[$sName])
+            || strlen($this->aParams[$sName]) == 0
+        ) {
             return $sDefault;
         }
 
-        if (!in_array($this->aParams[$sName], $aValues)) {
+        if (!in_array($this->aParams[$sName], $aValues, true)) {
             userError("Parameter '$sName' must be one of: ".join(', ', $aValues));
         }
 
@@ -106,21 +123,27 @@ class ParameterParser
         }
 
         foreach ($aLanguages as $sLanguage => $fLanguagePref) {
-            $aLangPrefOrder['name:'.$sLanguage] = 'name:'.$sLanguage;
+            $this->addNameTag($aLangPrefOrder, 'name:'.$sLanguage);
         }
-        $aLangPrefOrder['name'] = 'name';
-        $aLangPrefOrder['brand'] = 'brand';
+        $this->addNameTag($aLangPrefOrder, 'name');
+        $this->addNameTag($aLangPrefOrder, 'brand');
         foreach ($aLanguages as $sLanguage => $fLanguagePref) {
-            $aLangPrefOrder['official_name:'.$sLanguage] = 'official_name:'.$sLanguage;
-            $aLangPrefOrder['short_name:'.$sLanguage] = 'short_name:'.$sLanguage;
+            $this->addNameTag($aLangPrefOrder, 'official_name:'.$sLanguage);
+            $this->addNameTag($aLangPrefOrder, 'short_name:'.$sLanguage);
         }
-        $aLangPrefOrder['official_name'] = 'official_name';
-        $aLangPrefOrder['short_name'] = 'short_name';
-        $aLangPrefOrder['ref'] = 'ref';
-        $aLangPrefOrder['type'] = 'type';
+        $this->addNameTag($aLangPrefOrder, 'official_name');
+        $this->addNameTag($aLangPrefOrder, 'short_name');
+        $this->addNameTag($aLangPrefOrder, 'ref');
+        $this->addNameTag($aLangPrefOrder, 'type');
         return $aLangPrefOrder;
     }
 
+    private function addNameTag(&$aLangPrefOrder, $sTag)
+    {
+        $aLangPrefOrder[$sTag] = $sTag;
+        $aLangPrefOrder['_place_'.$sTag] = '_place_'.$sTag;
+    }
+
     public function hasSetAny($aParamNames)
     {
         foreach ($aParamNames as $sName) {

lib-php/Phrase.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -24,7 +32,7 @@ class Phrase
     }
 
     /**
-     * Get the orginal phrase of the string.
+     * Get the original phrase of the string.
      */
     public function getPhrase()
     {

lib-php/PlaceLookup.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -179,12 +187,12 @@ class PlaceLookup
             return null;
         }
 
-        $aResults = $this->lookup(array($iPlaceID => new Result($iPlaceID)));
+        $aResults = $this->lookup(array($iPlaceID => new Result($iPlaceID)), 0, 30, true);
 
         return empty($aResults) ? null : reset($aResults);
     }
 
-    public function lookup($aResults, $iMinRank = 0, $iMaxRank = 30)
+    public function lookup($aResults, $iMinRank = 0, $iMaxRank = 30, $bAllowLinked = false)
     {
         Debug::newFunction('Place lookup');
 
@@ -239,7 +247,9 @@ class PlaceLookup
             if ($this->sAllowedTypesSQLList) {
                 $sSQL .= 'AND placex.class in '.$this->sAllowedTypesSQLList;
             }
-            $sSQL .= '    AND linked_place_id is null ';
+            if (!$bAllowLinked) {
+                $sSQL .= '    AND linked_place_id is null ';
+            }
             $sSQL .= ' GROUP BY ';
             $sSQL .= '     osm_type, ';
             $sSQL .= '     osm_id, ';
@@ -340,7 +350,9 @@ class PlaceLookup
                     $sSQL .= '     null::text AS extra_place ';
                     $sSQL .= ' FROM (';
                     $sSQL .= '     SELECT place_id, ';    // interpolate the Tiger housenumbers here
-                    $sSQL .= '         ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) AS centroid, ';
+                    $sSQL .= '         CASE WHEN startnumber != endnumber';
+                    $sSQL .= '              THEN ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float)';
+                    $sSQL .= '              ELSE ST_LineInterpolatePoint(linegeo, 0.5) END AS centroid, ';
                     $sSQL .= '         parent_place_id, ';
                     $sSQL .= '         housenumber_for_place';
                     $sSQL .= '     FROM (';
@@ -397,7 +409,7 @@ class PlaceLookup
                 $sSQL .= '         CASE ';             // interpolate the housenumbers here
                 $sSQL .= '           WHEN startnumber != endnumber ';
                 $sSQL .= '           THEN ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) ';
-                $sSQL .= '           ELSE ST_LineInterpolatePoint(linegeo, 0.5) ';
+                $sSQL .= '           ELSE linegeo ';
                 $sSQL .= '         END as centroid, ';
                 $sSQL .= '         parent_place_id, ';
                 $sSQL .= '         housenumber_for_place ';
@@ -435,18 +447,14 @@ class PlaceLookup
 
             if ($this->bExtraTags) {
                 if ($aPlace['extra']) {
-                    $aPlace['sExtraTags'] = json_decode($aPlace['extra']);
+                    $aPlace['sExtraTags'] = json_decode($aPlace['extra'], true);
                 } else {
                     $aPlace['sExtraTags'] = (object) array();
                 }
             }
 
             if ($this->bNameDetails) {
-                if ($aPlace['names']) {
-                    $aPlace['sNameDetails'] = json_decode($aPlace['names']);
-                } else {
-                    $aPlace['sNameDetails'] = (object) array();
-                }
+                $aPlace['sNameDetails'] = $this->extractNames($aPlace['names']);
             }
 
             $aPlace['addresstype'] = ClassTypes\getLabelTag(
@@ -469,6 +477,33 @@ class PlaceLookup
         return $aResults;
     }
 
+
+    private function extractNames($sNames)
+    {
+        if (!$sNames) {
+            return (object) array();
+        }
+
+        $aFullNames = json_decode($sNames, true);
+        $aNames = array();
+
+        foreach ($aFullNames as $sKey => $sValue) {
+            if (strpos($sKey, '_place_') === 0) {
+                $sSubKey = substr($sKey, 7);
+                if (array_key_exists($sSubKey, $aFullNames)) {
+                    $aNames[$sKey] = $sValue;
+                } else {
+                    $aNames[$sSubKey] = $sValue;
+                }
+            } else {
+                $aNames[$sKey] = $sValue;
+            }
+        }
+
+        return $aNames;
+    }
+
+
     /* returns an array which will contain the keys
      *   aBoundingBox
      * and may also contain one or more of the keys
@@ -479,8 +514,6 @@ class PlaceLookup
      *   lat
      *   lon
      */
-
-
     public function getOutlines($iPlaceID, $fLon = null, $fLat = null, $fRadius = null, $fLonReverse = null, $fLatReverse = null)
     {
 
@@ -491,12 +524,7 @@ class PlaceLookup
 
         // Get the bounding box and outline polygon
         $sSQL = 'select place_id,0 as numfeatures,st_area(geometry) as area,';
-        if ($fLonReverse != null && $fLatReverse != null) {
-            $sSQL .= ' ST_Y(closest_point) as centrelat,';
-            $sSQL .= ' ST_X(closest_point) as centrelon,';
-        } else {
-            $sSQL .= ' ST_Y(centroid) as centrelat, ST_X(centroid) as centrelon,';
-        }
+        $sSQL .= ' ST_Y(centroid) as centrelat, ST_X(centroid) as centrelon,';
         $sSQL .= ' ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,';
         $sSQL .= ' ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon';
         if ($this->bIncludePolygonAsGeoJSON) {
@@ -511,19 +539,21 @@ class PlaceLookup
         if ($this->bIncludePolygonAsText) {
             $sSQL .= ',ST_AsText(geometry) as astext';
         }
+
+        $sSQL .= ' FROM (SELECT place_id';
         if ($fLonReverse != null && $fLatReverse != null) {
-            $sFrom = ' from (SELECT * , CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
-            $sFrom .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
-            $sFrom .=' ELSE centroid END AS closest_point';
-            $sFrom .= ' from placex where place_id = '.$iPlaceID.') as plx';
+            $sSQL .= ',CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
+            $sSQL .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
+            $sSQL .=' ELSE centroid END AS centroid';
         } else {
-            $sFrom = ' from placex where place_id = '.$iPlaceID;
+            $sSQL .= ',centroid';
         }
         if ($this->fPolygonSimplificationThreshold > 0) {
-            $sSQL .= ' from (select place_id,centroid,ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry'.$sFrom.') as plx';
+            $sSQL .= ',ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry';
         } else {
-            $sSQL .= $sFrom;
+            $sSQL .= ',geometry';
         }
+        $sSQL .= ' FROM placex where place_id = '.$iPlaceID.') as plx';
 
         $aPointPolygon = $this->oDB->getRow($sSQL, null, 'Could not get outline');
 

lib-php/Result.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 

lib-php/ReverseGeocode.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -56,12 +64,15 @@ class ReverseGeocode
     {
         Debug::newFunction('lookupInterpolation');
         $sSQL = 'SELECT place_id, parent_place_id, 30 as rank_search,';
-        $sSQL .= '  ST_LineLocatePoint(linegeo,'.$sPointSQL.') as fraction,';
-        $sSQL .= '  startnumber, endnumber, interpolationtype,';
+        $sSQL .= '  (CASE WHEN endnumber != startnumber';
+        $sSQL .= '        THEN (endnumber - startnumber) * ST_LineLocatePoint(linegeo,'.$sPointSQL.')';
+        $sSQL .= '        ELSE startnumber END) as fhnr,';
+        $sSQL .= '  startnumber, endnumber, step,';
         $sSQL .= '  ST_Distance(linegeo,'.$sPointSQL.') as distance';
         $sSQL .= ' FROM location_property_osmline';
         $sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', linegeo, '.$fSearchDiam.')';
-        $sSQL .= ' and indexed_status = 0 and startnumber is not NULL ';
+        $sSQL .= '       and indexed_status = 0 and startnumber is not NULL ';
+        $sSQL .= '       and parent_place_id != 0';
         $sSQL .= ' ORDER BY distance ASC limit 1';
         Debug::printSQL($sSQL);
 
@@ -178,14 +189,16 @@ class ReverseGeocode
         $sSQL .= '(select place_id, parent_place_id, rank_address, rank_search, country_code, geometry';
         $sSQL .= ' FROM placex';
         $sSQL .= ' WHERE ST_GeometryType(geometry) in (\'ST_Polygon\', \'ST_MultiPolygon\')';
-        $sSQL .= ' AND rank_address Between 5 AND ' .$iMaxRank;
+        // Ensure that query planner doesn't use the index on rank_search.
+        $sSQL .= ' AND coalesce(rank_search, 0) between 5 and ' .$iMaxRank;
+        $sSQL .= ' AND rank_address between 4 and 25'; // needed for index selection
         $sSQL .= ' AND geometry && '.$sPointSQL;
         $sSQL .= ' AND type != \'postcode\' ';
         $sSQL .= ' AND name is not null';
         $sSQL .= ' AND indexed_status = 0 and linked_place_id is null';
-        $sSQL .= ' ORDER BY rank_address DESC LIMIT 50 ) as a';
-        $sSQL .= ' WHERE ST_CONTAINS(geometry, '.$sPointSQL.' )';
-        $sSQL .= ' ORDER BY rank_address DESC LIMIT 1';
+        $sSQL .= ' ORDER BY rank_search DESC LIMIT 50 ) as a';
+        $sSQL .= ' WHERE ST_Contains(geometry, '.$sPointSQL.' )';
+        $sSQL .= ' ORDER BY rank_search DESC LIMIT 1';
         Debug::printSQL($sSQL);
 
         $aPoly = $this->oDB->getRow($sSQL, null, 'Could not determine polygon containing the point.');
@@ -197,7 +210,7 @@ class ReverseGeocode
             $iRankSearch = $aPoly['rank_search'];
             $iPlaceID = $aPoly['place_id'];
 
-            if ($iRankAddress != $iMaxRank) {
+            if ($iRankSearch != $iMaxRank) {
                 $sSQL = 'SELECT place_id FROM ';
                 $sSQL .= '(SELECT place_id, rank_search, country_code, geometry,';
                 $sSQL .= ' ST_distance('.$sPointSQL.', geometry) as distance';
@@ -255,7 +268,7 @@ class ReverseGeocode
             // starts if the search is on POI or street level,
             // searches for the nearest POI or street,
             // if a street is found and a POI is searched for,
-            // the nearest POI which the found street is a parent of is choosen.
+            // the nearest POI which the found street is a parent of is chosen.
             $sSQL = 'select place_id,parent_place_id,rank_address,country_code,';
             $sSQL .= ' ST_distance('.$sPointSQL.', geometry) as distance';
             $sSQL .= ' FROM ';
@@ -319,9 +332,9 @@ class ReverseGeocode
                     && $this->iMaxRank >= 28
                 ) {
                     $sSQL = 'SELECT place_id,parent_place_id,30 as rank_search,';
-                    $sSQL .= 'ST_LineLocatePoint(linegeo,'.$sPointSQL.') as fraction,';
-                    $sSQL .= 'ST_distance('.$sPointSQL.', linegeo) as distance,';
-                    $sSQL .= 'startnumber,endnumber,interpolationtype';
+                    $sSQL .= '      (endnumber - startnumber) * ST_LineLocatePoint(linegeo,'.$sPointSQL.') as fhnr,';
+                    $sSQL .= '      startnumber, endnumber, step,';
+                    $sSQL .= '      ST_Distance('.$sPointSQL.', linegeo) as distance';
                     $sSQL .= ' FROM location_property_tiger WHERE parent_place_id = '.$oResult->iId;
                     $sSQL .= ' AND ST_DWithin('.$sPointSQL.', linegeo, 0.001)';
                     $sSQL .= ' ORDER BY distance ASC limit 1';
@@ -333,7 +346,11 @@ class ReverseGeocode
                     if ($aPlaceTiger) {
                         $aPlace = $aPlaceTiger;
                         $oResult = new Result($aPlaceTiger['place_id'], Result::TABLE_TIGER);
-                        $oResult->iHouseNumber = closestHouseNumber($aPlaceTiger);
+                        $iRndNum = max(0, round($aPlaceTiger['fhnr'] / $aPlaceTiger['step']) * $aPlaceTiger['step']);
+                        $oResult->iHouseNumber = $aPlaceTiger['startnumber'] + $iRndNum;
+                        if ($oResult->iHouseNumber > $aPlaceTiger['endnumber']) {
+                            $oResult->iHouseNumber = $aPlaceTiger['endnumber'];
+                        }
                         $iRankAddress = 30;
                     }
                 }
@@ -345,7 +362,7 @@ class ReverseGeocode
                     // We can't reliably go from the closest street to an
                     // interpolation line because the closest interpolation
                     // may have a different street segments as a parent.
-                    // Therefore allow an interpolation line to take precendence
+                    // Therefore allow an interpolation line to take precedence
                     // even when the street is closer.
                     $fDistance = $iRankAddress < 28 ? 0.001 : $aPlace['distance'];
                 }
@@ -355,7 +372,11 @@ class ReverseGeocode
 
                 if ($aHouse) {
                     $oResult = new Result($aHouse['place_id'], Result::TABLE_OSMLINE);
-                    $oResult->iHouseNumber = closestHouseNumber($aHouse);
+                    $iRndNum = max(0, round($aHouse['fhnr'] / $aHouse['step']) * $aHouse['step']);
+                    $oResult->iHouseNumber = $aHouse['startnumber'] + $iRndNum;
+                    if ($oResult->iHouseNumber > $aHouse['endnumber']) {
+                        $oResult->iHouseNumber = $aHouse['endnumber'];
+                    }
                     $aPlace = $aHouse;
                 }
             }

lib-php/SearchContext.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 

lib-php/SearchDescription.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -228,7 +236,7 @@ class SearchDescription
      * Add the given full-word token to the list of terms to search for in the
      * name.
      *
-     * @param interger iId    ID of term to add.
+     * @param integer iId    ID of term to add.
      * @param bool bRareName  True if the term is infrequent enough to not
      *                        require other constraints for efficient search.
      */
@@ -256,8 +264,10 @@ class SearchDescription
     {
         if (empty($this->aName)) {
             $this->bNameNeedsAddress = $bNeedsAddress;
+        } elseif ($bSearchable && count($this->aName) >= 2) {
+            $this->bNameNeedsAddress = false;
         } else {
-            $this->bNameNeedsAddress |= $bNeedsAddress;
+            $this->bNameNeedsAddress &= $bNeedsAddress;
         }
         if ($bSearchable) {
             $this->aName[$iId] = $iId;
@@ -377,7 +387,7 @@ class SearchDescription
      *
      * @return mixed[] An array with two fields: IDs contains the list of
      *                 matching place IDs and houseNumber the houseNumber
-     *                 if appicable or -1 if not.
+     *                 if applicable or -1 if not.
      */
     public function query(&$oDB, $iMinRank, $iMaxRank, $iLimit)
     {
@@ -412,28 +422,6 @@ class SearchDescription
                 $iLimit
             );
 
-            // Now search for housenumber, if housenumber provided. Can be zero.
-            if (($this->sHouseNumber || $this->sHouseNumber === '0') && !empty($aResults)) {
-                $aHnResults = $this->queryHouseNumber($oDB, $aResults);
-
-                // Downgrade the rank of the street results, they are missing
-                // the housenumber. Also drop POI places (rank 30) here, they
-                // cannot be a parent place and therefore must not be shown
-                // as a result for a search with a missing housenumber.
-                foreach ($aResults as $oRes) {
-                    if ($oRes->iAddressRank < 28) {
-                        if ($oRes->iAddressRank >= 26) {
-                            $oRes->iResultRank++;
-                        } else {
-                            $oRes->iResultRank += 2;
-                        }
-                        $aHnResults[$oRes->iId] = $oRes;
-                    }
-                }
-
-                $aResults = $aHnResults;
-            }
-
             // finally get POIs if requested
             if ($this->sClass && !empty($aResults)) {
                 $aResults = $this->queryPoiByOperator($oDB, $aResults, $iLimit);
@@ -579,41 +567,6 @@ class SearchDescription
         $aTerms = array();
         $aOrder = array();
 
-        // Sort by existence of the requested house number but only if not
-        // too many results are expected for the street, i.e. if the result
-        // will be narrowed down by an address. Remember that with ordering
-        // every single result has to be checked.
-        if ($this->sHouseNumber && ($this->bRareName || !empty($this->aAddress) || $this->sPostcode)) {
-            $sHouseNumberRegex = '\\\\m'.$this->sHouseNumber.'\\\\M';
-
-            // Housenumbers on streets and places.
-            $sChildHnr = 'SELECT * FROM placex WHERE parent_place_id = search_name.place_id';
-            $sChildHnr .= "    AND housenumber ~* E'".$sHouseNumberRegex."'";
-            // Interpolations on streets and places.
-            if (preg_match('/^[0-9]+$/', $this->sHouseNumber)) {
-                $sIpolHnr = 'SELECT * FROM location_property_osmline ';
-                $sIpolHnr .= 'WHERE parent_place_id = search_name.place_id ';
-                $sIpolHnr .= '  AND startnumber is not NULL';
-                $sIpolHnr .= '    AND '.$this->sHouseNumber.'>=startnumber ';
-                $sIpolHnr .= '    AND '.$this->sHouseNumber.'<=endnumber ';
-            } else {
-                $sIpolHnr = false;
-            }
-            // Housenumbers on the object iteself for unlisted places.
-            $sSelfHnr = 'SELECT * FROM placex WHERE place_id = search_name.place_id';
-            $sSelfHnr .= "    AND housenumber ~* E'".$sHouseNumberRegex."'";
-
-            $sSql = '(CASE WHEN address_rank = 30 THEN EXISTS('.$sSelfHnr.') ';
-            $sSql .= ' ELSE EXISTS('.$sChildHnr.') ';
-            if ($sIpolHnr) {
-                $sSql .= 'OR EXISTS('.$sIpolHnr.') ';
-            }
-            $sSql .= 'END) DESC';
-
-
-            $aOrder[] = $sSql;
-        }
-
         if (!empty($this->aName)) {
             $aTerms[] = 'name_vector @> '.$oDB->getArraySQL($this->aName);
         }
@@ -659,10 +612,6 @@ class SearchDescription
             $aTerms[] = 'centroid && '.$this->oContext->sqlViewboxSmall;
         }
 
-        if ($this->oContext->hasNearPoint()) {
-            $aOrder[] = $this->oContext->distanceSQL('centroid');
-        }
-
         if ($this->sHouseNumber) {
             $sImportanceSQL = '- abs(26 - address_rank) + 3';
         } else {
@@ -685,122 +634,128 @@ class SearchDescription
             $sExactMatchSQL = '0::int as exactmatch';
         }
 
-        if ($this->sHouseNumber || $this->sClass) {
-            $iLimit = 40;
+        if (empty($aTerms)) {
+            return array();
         }
 
-        $aResults = array();
+        if ($this->hasHousenumber()) {
+            $sHouseNumberRegex = $oDB->getDBQuoted('\\\\m'.$this->sHouseNumber.'\\\\M');
 
-        if (!empty($aTerms)) {
-            $sSQL = 'SELECT place_id, address_rank,'.$sExactMatchSQL;
+            // Housenumbers on streets and places.
+            $sPlacexSql = 'SELECT array_agg(place_id) FROM placex';
+            $sPlacexSql .= ' WHERE parent_place_id = sin.place_id AND sin.address_rank < 30';
+            $sPlacexSql .= $this->oContext->excludeSQL(' AND place_id');
+            $sPlacexSql .= '       and housenumber ~* E'.$sHouseNumberRegex;
+
+            // Interpolations on streets and places.
+            $sInterpolSql = 'null';
+            $sTigerSql = 'null';
+            if (preg_match('/^[0-9]+$/', $this->sHouseNumber)) {
+                $sIpolHnr = 'WHERE parent_place_id = sin.place_id ';
+                $sIpolHnr .= '  AND startnumber is not NULL AND sin.address_rank < 30';
+                $sIpolHnr .= '  AND '.$this->sHouseNumber.' between startnumber and endnumber';
+                $sIpolHnr .= '  AND ('.$this->sHouseNumber.' - startnumber) % step = 0';
+
+                $sInterpolSql = 'SELECT array_agg(place_id) FROM location_property_osmline '.$sIpolHnr;
+                if (CONST_Use_US_Tiger_Data) {
+                    $sTigerSql = 'SELECT array_agg(place_id) FROM location_property_tiger '.$sIpolHnr;
+                    $sTigerSql .= "      and sin.country_code = 'us'";
+                }
+            }
+
+            if ($this->sClass) {
+                $iLimit = 40;
+            }
+
+            $sSelfHnr = 'SELECT * FROM placex WHERE place_id = search_name.place_id';
+            $sSelfHnr .= '    AND housenumber ~* E'.$sHouseNumberRegex;
+
+            $aTerms[] = '(address_rank < 30 or exists('.$sSelfHnr.'))';
+
+
+            $sSQL = 'SELECT sin.*, ';
+            $sSQL .=        '('.$sPlacexSql.') as placex_hnr, ';
+            $sSQL .=        '('.$sInterpolSql.') as interpol_hnr, ';
+            $sSQL .=        '('.$sTigerSql.') as tiger_hnr ';
+            $sSQL .= ' FROM (';
+            $sSQL .= '    SELECT place_id, address_rank, country_code,'.$sExactMatchSQL.',';
+            $sSQL .= '            CASE WHEN importance = 0 OR importance IS NULL';
+            $sSQL .= '               THEN 0.75001-(search_rank::float/40) ELSE importance END as importance';
+            $sSQL .= '     FROM search_name';
+            $sSQL .= '     WHERE '.join(' and ', $aTerms);
+            $sSQL .= '     ORDER BY '.join(', ', $aOrder);
+            $sSQL .= '     LIMIT 40000';
+            $sSQL .= ') as sin';
+            $sSQL .= ' ORDER BY address_rank = 30 desc, placex_hnr, interpol_hnr, tiger_hnr,';
+            $sSQL .= '          importance';
+            $sSQL .= ' LIMIT '.$iLimit;
+        } else {
+            if ($this->sClass) {
+                $iLimit = 40;
+            }
+
+            $sSQL = 'SELECT place_id, address_rank, '.$sExactMatchSQL;
             $sSQL .= ' FROM search_name';
             $sSQL .= ' WHERE '.join(' and ', $aTerms);
             $sSQL .= ' ORDER BY '.join(', ', $aOrder);
             $sSQL .= ' LIMIT '.$iLimit;
-
-            Debug::printSQL($sSQL);
-
-            $aDBResults = $oDB->getAll($sSQL, null, 'Could not get places for search terms.');
-
-            foreach ($aDBResults as $aResult) {
-                $oResult = new Result($aResult['place_id']);
-                $oResult->iExactMatches = $aResult['exactmatch'];
-                $oResult->iAddressRank = $aResult['address_rank'];
-                $aResults[$aResult['place_id']] = $oResult;
-            }
         }
 
-        return $aResults;
-    }
-
-    private function queryHouseNumber(&$oDB, $aRoadPlaceIDs)
-    {
-        $aResults = array();
-        $sRoadPlaceIDs = Result::joinIdsByTableMaxRank(
-            $aRoadPlaceIDs,
-            Result::TABLE_PLACEX,
-            27
-        );
-        $sPOIPlaceIDs = Result::joinIdsByTableMinRank(
-            $aRoadPlaceIDs,
-            Result::TABLE_PLACEX,
-            30
-        );
-
-        $aIDCondition = array();
-        if ($sRoadPlaceIDs) {
-            $aIDCondition[] = 'parent_place_id in ('.$sRoadPlaceIDs.')';
-        }
-        if ($sPOIPlaceIDs) {
-            $aIDCondition[] = 'place_id in ('.$sPOIPlaceIDs.')';
-        }
-
-        if (empty($aIDCondition)) {
-            return $aResults;
-        }
-
-        $sHouseNumberRegex = '\\\\m'.$this->sHouseNumber.'\\\\M';
-        $sSQL = 'SELECT place_id FROM placex WHERE';
-        $sSQL .= "  housenumber ~* E'".$sHouseNumberRegex."'";
-        $sSQL .= ' AND ('.join(' OR ', $aIDCondition).')';
-        $sSQL .= $this->oContext->excludeSQL(' AND place_id');
-
         Debug::printSQL($sSQL);
 
-        // XXX should inherit the exactMatches from its parent
-        foreach ($oDB->getCol($sSQL) as $iPlaceId) {
-            $aResults[$iPlaceId] = new Result($iPlaceId);
-        }
+        $aDBResults = $oDB->getAll($sSQL, null, 'Could not get places for search terms.');
 
-        $bIsIntHouseNumber= (bool) preg_match('/[0-9]+/', $this->sHouseNumber);
-        $iHousenumber = intval($this->sHouseNumber);
-        if ($bIsIntHouseNumber && $sRoadPlaceIDs && empty($aResults)) {
-            // if nothing found, search in the interpolation line table
-            $sSQL = 'SELECT distinct place_id FROM location_property_osmline';
-            $sSQL .= ' WHERE startnumber is not NULL';
-            $sSQL .= '  AND parent_place_id in ('.$sRoadPlaceIDs.') AND (';
-            if ($iHousenumber % 2 == 0) {
-                // If housenumber is even, look for housenumber in streets
-                // with interpolationtype even or all.
-                $sSQL .= "interpolationtype='even'";
-            } else {
-                // Else look for housenumber with interpolationtype odd or all.
-                $sSQL .= "interpolationtype='odd'";
+        $aResults = array();
+
+        foreach ($aDBResults as $aResult) {
+            $oResult = new Result($aResult['place_id']);
+            $oResult->iExactMatches = $aResult['exactmatch'];
+            $oResult->iAddressRank = $aResult['address_rank'];
+
+            $bNeedResult = true;
+            if ($this->hasHousenumber() && $aResult['address_rank'] < 30) {
+                if ($aResult['placex_hnr']) {
+                    foreach (explode(',', substr($aResult['placex_hnr'], 1, -1)) as $sPlaceID) {
+                        $iPlaceID = intval($sPlaceID);
+                        $oHnrResult = new Result($iPlaceID);
+                        $oHnrResult->iExactMatches = $aResult['exactmatch'];
+                        $oHnrResult->iAddressRank = 30;
+                        $aResults[$iPlaceID] = $oHnrResult;
+                        $bNeedResult = false;
+                    }
+                }
+                if ($aResult['interpol_hnr']) {
+                    foreach (explode(',', substr($aResult['interpol_hnr'], 1, -1)) as $sPlaceID) {
+                        $iPlaceID = intval($sPlaceID);
+                        $oHnrResult = new Result($iPlaceID, Result::TABLE_OSMLINE);
+                        $oHnrResult->iExactMatches = $aResult['exactmatch'];
+                        $oHnrResult->iAddressRank = 30;
+                        $oHnrResult->iHouseNumber = intval($this->sHouseNumber);
+                        $aResults[$iPlaceID] = $oHnrResult;
+                        $bNeedResult = false;
+                    }
+                }
+                if ($aResult['tiger_hnr']) {
+                    foreach (explode(',', substr($aResult['tiger_hnr'], 1, -1)) as $sPlaceID) {
+                        $iPlaceID = intval($sPlaceID);
+                        $oHnrResult = new Result($iPlaceID, Result::TABLE_TIGER);
+                        $oHnrResult->iExactMatches = $aResult['exactmatch'];
+                        $oHnrResult->iAddressRank = 30;
+                        $oHnrResult->iHouseNumber = intval($this->sHouseNumber);
+                        $aResults[$iPlaceID] = $oHnrResult;
+                        $bNeedResult = false;
+                    }
+                }
+
+                if ($aResult['address_rank'] < 26) {
+                    $oResult->iResultRank += 2;
+                } else {
+                    $oResult->iResultRank++;
+                }
             }
-            $sSQL .= " or interpolationtype='all') and ";
-            $sSQL .= $iHousenumber.'>=startnumber and ';
-            $sSQL .= $iHousenumber.'<=endnumber';
-            $sSQL .= $this->oContext->excludeSQL(' AND place_id');
 
-            Debug::printSQL($sSQL);
-
-            foreach ($oDB->getCol($sSQL) as $iPlaceId) {
-                $oResult = new Result($iPlaceId, Result::TABLE_OSMLINE);
-                $oResult->iHouseNumber = $iHousenumber;
-                $aResults[$iPlaceId] = $oResult;
-            }
-        }
-
-        // If nothing found then search in Tiger data (location_property_tiger)
-        if (CONST_Use_US_Tiger_Data && $sRoadPlaceIDs && $bIsIntHouseNumber && empty($aResults)) {
-            $sSQL = 'SELECT place_id FROM location_property_tiger';
-            $sSQL .= ' WHERE parent_place_id in ('.$sRoadPlaceIDs.') and (';
-            if ($iHousenumber % 2 == 0) {
-                $sSQL .= "interpolationtype='even'";
-            } else {
-                $sSQL .= "interpolationtype='odd'";
-            }
-            $sSQL .= " or interpolationtype='all') and ";
-            $sSQL .= $iHousenumber.'>=startnumber and ';
-            $sSQL .= $iHousenumber.'<=endnumber';
-            $sSQL .= $this->oContext->excludeSQL(' AND place_id');
-
-            Debug::printSQL($sSQL);
-
-            foreach ($oDB->getCol($sSQL) as $iPlaceId) {
-                $oResult = new Result($iPlaceId, Result::TABLE_TIGER);
-                $oResult->iHouseNumber = $iHousenumber;
-                $aResults[$iPlaceId] = $oResult;
+            if ($bNeedResult) {
+                $aResults[$aResult['place_id']] = $oResult;
             }
         }
 
@@ -852,6 +807,7 @@ class SearchDescription
                 $sSQL = 'SELECT geometry FROM placex';
                 $sSQL .= " WHERE place_id in ($sPlaceIDs)";
                 $sSQL .= "   AND rank_search < $iMaxRank + 5";
+                $sSQL .= '   AND ST_Area(Box2d(geometry)) < 20';
                 $sSQL .= "   AND ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon')";
                 $sSQL .= ' ORDER BY rank_search ASC ';
                 $sSQL .= ' LIMIT 1';

lib-php/SearchPosition.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 

lib-php/Shell.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 

lib-php/SimpleWordList.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -112,13 +120,18 @@ class SimpleWordList
         return array_slice($aWordSets, 0, SimpleWordList::MAX_WORDSETS);
     }
 
+    /**
+     * Custom search routine which takes two arrays. The array with the fewest
+     * items wins. If same number of items then the one with the longest first
+     * element wins.
+     */
     public static function cmpByArraylen($aA, $aB)
     {
         $iALen = count($aA);
         $iBLen = count($aB);
 
         if ($iALen == $iBLen) {
-            return 0;
+            return strlen($aB[0]) <=> strlen($aA[0]);
         }
 
         return ($iALen < $iBLen) ? -1 : 1;

lib-php/SpecialSearchOperator.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 

lib-php/Status.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 

lib-php/TokenCountry.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim\Token;
 

lib-php/TokenHousenumber.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim\Token;
 
@@ -58,7 +66,7 @@ class HouseNumber
         // up of numbers, add a penalty
         $iSearchCost = 1;
         if (preg_match('/\\d/', $this->sToken) === 0
-            || preg_match_all('/[^0-9]/', $this->sToken, $aMatches) > 2) {
+            || preg_match_all('/[^0-9 ]/', $this->sToken, $aMatches) > 3) {
             $iSearchCost += strlen($this->sToken) - 1;
         }
         if (!$oSearch->hasOperator(\Nominatim\Operator::NONE)) {

lib-php/TokenList.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 

lib-php/TokenPartial.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim\Token;
 

lib-php/TokenPostcode.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim\Token;
 
@@ -9,7 +17,7 @@ class Postcode
 {
     /// Database word id, if available.
     private $iId;
-    /// Full nomralized postcode (upper cased).
+    /// Full normalized postcode (upper cased).
     private $sPostcode;
     // Optional country code the postcode belongs to (currently unused).
     private $sCountryCode;
@@ -17,7 +25,12 @@ class Postcode
     public function __construct($iId, $sPostcode, $sCountryCode = '')
     {
         $this->iId = $iId;
-        $this->sPostcode = $sPostcode;
+        $iSplitPos = strpos($sPostcode, '@');
+        if ($iSplitPos === false) {
+            $this->sPostcode = $sPostcode;
+        } else {
+            $this->sPostcode = substr($sPostcode, 0, $iSplitPos);
+        }
         $this->sCountryCode = empty($sCountryCode) ? '' : $sCountryCode;
     }
 

lib-php/TokenSpecialTerm.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim\Token;
 
@@ -61,19 +69,31 @@ class SpecialTerm
      */
     public function extendSearch($oSearch, $oPosition)
     {
-        $iSearchCost = 2;
+        $iSearchCost = 0;
 
         $iOp = $this->iOperator;
         if ($iOp == \Nominatim\Operator::NONE) {
-            if ($oSearch->hasName() || $oSearch->getContext()->isBoundedSearch()) {
+            if ($oPosition->isFirstToken()
+                || $oSearch->hasName()
+                || $oSearch->getContext()->isBoundedSearch()
+            ) {
                 $iOp = \Nominatim\Operator::NAME;
+                $iSearchCost += 3;
             } else {
                 $iOp = \Nominatim\Operator::NEAR;
-                $iSearchCost += 2;
+                $iSearchCost += 4;
+                if (!$oPosition->isFirstToken()) {
+                    $iSearchCost += 3;
+                }
             }
-        } elseif (!$oPosition->isFirstToken() && !$oPosition->isLastToken()) {
+        } elseif ($oPosition->isFirstToken()) {
             $iSearchCost += 2;
+        } elseif ($oPosition->isLastToken()) {
+            $iSearchCost += 4;
+        } else {
+            $iSearchCost += 6;
         }
+
         if ($oSearch->hasHousenumber()) {
             $iSearchCost ++;
         }

lib-php/TokenWord.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim\Token;
 
@@ -54,7 +62,7 @@ class Word
     public function extendSearch($oSearch, $oPosition)
     {
         // Full words can only be a name if they appear at the beginning
-        // of the phrase. In structured search the name must forcably in
+        // of the phrase. In structured search the name must forcibly in
         // the first phrase. In unstructured search it may be in a later
         // phrase when the first phrase is a house number.
         if ($oSearch->hasName()

lib-php/admin/export.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
     @define('CONST_LibDir', dirname(dirname(__FILE__)));
     // Script to extract structured city and street data
     // from a running nominatim instance as CSV data

lib-php/admin/warm.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 @define('CONST_LibDir', dirname(dirname(__FILE__)));
 
 require_once(CONST_LibDir.'/init-cmd.php');

lib-php/cmd.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 require_once(CONST_LibDir.'/Shell.php');
 
@@ -98,7 +106,7 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
                 showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is missing');
             }
             if ($aCounts[$aLine[0]] > $aLine[3]) {
-                showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is pressent too many times');
+                showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is present too many times');
             }
             if ($aLine[6] == 'bool' && !array_key_exists($aLine[0], $aResult)) {
                 $aResult[$aLine[0]] = false;

lib-php/dotenv_loader.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 require('Symfony/Component/Dotenv/autoload.php');
 

lib-php/init-cmd.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 require_once('init.php');
 require_once('cmd.php');

lib-php/init-website.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 require_once('init.php');
 require_once('ParameterParser.php');
@@ -18,7 +26,7 @@ function userError($sMsg)
 
 function exception_handler_json($exception)
 {
-    http_response_code($exception->getCode());
+    http_response_code($exception->getCode() == 0 ? 500 : $exception->getCode());
     header('Content-type: application/json; charset=utf-8');
     include(CONST_LibDir.'/template/error-json.php');
     exit();
@@ -26,7 +34,7 @@ function exception_handler_json($exception)
 
 function exception_handler_xml($exception)
 {
-    http_response_code($exception->getCode());
+    http_response_code($exception->getCode() == 0 ? 500 : $exception->getCode());
     header('Content-type: text/xml; charset=utf-8');
     echo '<?xml version="1.0" encoding="UTF-8" ?>'."\n";
     include(CONST_LibDir.'/template/error-xml.php');

lib-php/init.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 require_once(CONST_LibDir.'/lib.php');
 require_once(CONST_LibDir.'/DB.php');

lib-php/lib.php

@@ -1,9 +1,17 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 function loadSettings($sProjectDir)
 {
     @define('CONST_InstallDir', $sProjectDir);
-    // Temporary hack to set the direcory via environment instead of
+    // Temporary hack to set the directory via environment instead of
     // the installed scripts. Neither setting is part of the official
     // set of settings.
     defined('CONST_ConfigDir') or define('CONST_ConfigDir', $_SERVER['NOMINATIM_CONFIGDIR']);
@@ -198,24 +206,34 @@ function parseLatLon($sQuery)
     return array($sFound, $fQueryLat, $fQueryLon);
 }
 
-function closestHouseNumber($aRow)
+function addressRankToGeocodeJsonType($iAddressRank)
 {
-    $fHouse = $aRow['startnumber']
-                + ($aRow['endnumber'] - $aRow['startnumber']) * $aRow['fraction'];
-
-    switch ($aRow['interpolationtype']) {
-        case 'odd':
-            $iHn = (int)($fHouse/2) * 2 + 1;
-            break;
-        case 'even':
-            $iHn = (int)(round($fHouse/2)) * 2;
-            break;
-        default:
-            $iHn = (int)(round($fHouse));
-            break;
+    if ($iAddressRank >= 29 && $iAddressRank <= 30) {
+        return 'house';
+    }
+    if ($iAddressRank >= 26 && $iAddressRank < 28) {
+        return 'street';
+    }
+    if ($iAddressRank >= 22 && $iAddressRank < 26) {
+        return 'locality';
+    }
+    if ($iAddressRank >= 17 && $iAddressRank < 22) {
+        return 'district';
+    }
+    if ($iAddressRank >= 13 && $iAddressRank < 17) {
+        return 'city';
+    }
+    if ($iAddressRank >= 10 && $iAddressRank < 13) {
+        return 'county';
+    }
+    if ($iAddressRank >= 5 && $iAddressRank < 10) {
+        return 'state';
+    }
+    if ($iAddressRank >= 4 && $iAddressRank < 5) {
+        return 'country';
     }
 
-    return max(min($aRow['endnumber'], $iHn), $aRow['startnumber']);
+    return 'locality';
 }
 
 if (!function_exists('array_key_last')) {

lib-php/log.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 
 function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array())
@@ -70,7 +78,7 @@ function logEnd(&$oDB, $hLog, $iNumResults)
 
     if (CONST_Log_DB) {
         $aEndTime = explode('.', $fEndTime);
-        if (!$aEndTime[1]) {
+        if (!isset($aEndTime[1])) {
             $aEndTime[1] = '0';
         }
         $sEndTime = date('Y-m-d H:i:s', $aEndTime[0]).'.'.$aEndTime[1];

lib-php/output.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 
 function formatOSMType($sType, $bIncludeExternal = true)

lib-php/setup_functions.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 function getOsm2pgsqlBinary()
 {

lib-php/template/address-geocodejson.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 // https://github.com/geocoders/geocodejson-spec/
 
@@ -28,7 +36,10 @@ if (empty($aPlace)) {
         $aFilteredPlaces['properties']['geocoding']['osm_id'] = $aPlace['osm_id'];
     }
 
-    $aFilteredPlaces['properties']['geocoding']['type'] = $aPlace['type'];
+    $aFilteredPlaces['properties']['geocoding']['osm_key'] = $aPlace['class'];
+    $aFilteredPlaces['properties']['geocoding']['osm_value'] = $aPlace['type'];
+
+    $aFilteredPlaces['properties']['geocoding']['type'] = addressRankToGeocodeJsonType($aPlace['rank_address']);
 
     $aFilteredPlaces['properties']['geocoding']['accuracy'] = (int) $fDistance;
 
@@ -48,7 +59,7 @@ if (empty($aPlace)) {
     }
 
     if (isset($aPlace['asgeojson'])) {
-        $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson']);
+        $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson'], true);
     } else {
         $aFilteredPlaces['geometry'] = array(
                                         'type' => 'Point',

lib-php/template/address-geojson.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 $aFilteredPlaces = array();
 
@@ -57,7 +65,7 @@ if (empty($aPlace)) {
     }
 
     if (isset($aPlace['asgeojson'])) {
-        $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson']);
+        $aFilteredPlaces['geometry'] = json_decode($aPlace['asgeojson'], true);
     } else {
         $aFilteredPlaces['geometry'] = array(
                                         'type' => 'Point',

lib-php/template/address-json.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 $aFilteredPlaces = array();
 
@@ -55,7 +63,7 @@ if (empty($aPlace)) {
     }
 
     if (isset($aPlace['asgeojson'])) {
-        $aFilteredPlaces['geojson'] = json_decode($aPlace['asgeojson']);
+        $aFilteredPlaces['geojson'] = json_decode($aPlace['asgeojson'], true);
     }
 
     if (isset($aPlace['assvg'])) {

lib-php/template/address-xml.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 header('content-type: text/xml; charset=UTF-8');
 
 echo '<';

lib-php/template/details-json.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 $aPlaceDetails = array();
 
@@ -40,7 +48,7 @@ $aPlaceDetails['centroid'] = array(
                               'coordinates' => array( (float) $aPointDetails['lon'], (float) $aPointDetails['lat'] )
                              );
 
-$aPlaceDetails['geometry'] = json_decode($aPointDetails['asgeojson']);
+$aPlaceDetails['geometry'] = json_decode($aPointDetails['asgeojson'], true);
 
 $funcMapAddressLine = function ($aFull) {
     return array(

lib-php/template/error-json.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
     $error = array(
               'code' => $exception->getCode(),
               'message' => $exception->getMessage()

lib-php/template/search-batch-json.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 $aOutput = array();
 $aOutput['licence'] = 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright';
@@ -52,7 +60,7 @@ foreach ($aBatchResults as $aSearchResults) {
         }
 
         if (isset($aPointDetails['asgeojson'])) {
-            $aPlace['geojson'] = json_decode($aPointDetails['asgeojson']);
+            $aPlace['geojson'] = json_decode($aPointDetails['asgeojson'], true);
         }
 
         if (isset($aPointDetails['assvg'])) {

lib-php/template/search-geocodejson.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 $aFilteredPlaces = array();
 foreach ($aSearchResults as $iResNum => $aPointDetails) {
@@ -17,8 +25,10 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
         $aPlace['properties']['geocoding']['osm_type'] = $sOSMType;
         $aPlace['properties']['geocoding']['osm_id'] = $aPointDetails['osm_id'];
     }
+    $aPlace['properties']['geocoding']['osm_key'] = $aPointDetails['class'];
+    $aPlace['properties']['geocoding']['osm_value'] = $aPointDetails['type'];
 
-    $aPlace['properties']['geocoding']['type'] = $aPointDetails['type'];
+    $aPlace['properties']['geocoding']['type'] = addressRankToGeocodeJsonType($aPointDetails['rank_address']);
 
     $aPlace['properties']['geocoding']['label'] = $aPointDetails['langaddress'];
 
@@ -36,7 +46,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
     }
 
     if (isset($aPointDetails['asgeojson'])) {
-        $aPlace['geometry'] = json_decode($aPointDetails['asgeojson']);
+        $aPlace['geometry'] = json_decode($aPointDetails['asgeojson'], true);
     } else {
         $aPlace['geometry'] = array(
                                'type' => 'Point',

lib-php/template/search-geojson.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 $aFilteredPlaces = array();
 foreach ($aSearchResults as $iResNum => $aPointDetails) {
@@ -46,7 +54,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
     }
 
     if (isset($aPointDetails['asgeojson'])) {
-        $aPlace['geometry'] = json_decode($aPointDetails['asgeojson']);
+        $aPlace['geometry'] = json_decode($aPointDetails['asgeojson'], true);
     } else {
         $aPlace['geometry'] = array(
                                'type' => 'Point',

lib-php/template/search-json.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 $aFilteredPlaces = array();
 foreach ($aSearchResults as $iResNum => $aPointDetails) {
@@ -45,7 +53,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
     }
 
     if (isset($aPointDetails['asgeojson'])) {
-        $aPlace['geojson'] = json_decode($aPointDetails['asgeojson']);
+        $aPlace['geojson'] = json_decode($aPointDetails['asgeojson'], true);
     }
 
     if (isset($aPointDetails['assvg'])) {

lib-php/template/search-xml.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 header('content-type: text/xml; charset=UTF-8');
 
 echo '<';

lib-php/tokenizer/icu_tokenizer.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 
@@ -149,7 +157,8 @@ class Tokenizer
         $sSQL = 'SELECT word_id, word_token, type, word,';
         $sSQL .= "      info->>'op' as operator,";
         $sSQL .= "      info->>'class' as class, info->>'type' as ctype,";
-        $sSQL .= "      info->>'count' as count";
+        $sSQL .= "      info->>'count' as count,";
+        $sSQL .= "      info->>'lookup' as lookup";
         $sSQL .= ' FROM word WHERE word_token in (';
         $sSQL .= join(',', $this->oDB->getDBQuotedList($aTokens)).')';
 
@@ -171,7 +180,8 @@ class Tokenizer
                     }
                     break;
                 case 'H':  // house number tokens
-                    $oValidTokens->addToken($sTok, new Token\HouseNumber($iId, $aWord['word_token']));
+                    $sLookup = $aWord['lookup'] ?? $aWord['word_token'];
+                    $oValidTokens->addToken($sTok, new Token\HouseNumber($iId, $sLookup));
                     break;
                 case 'P':  // postcode tokens
                     // Postcodes are not normalized, so they may have content
@@ -180,13 +190,17 @@ class Tokenizer
                     if ($aWord['word'] !== null
                         && pg_escape_string($aWord['word']) == $aWord['word']
                     ) {
-                        $sNormPostcode = $this->normalizeString($aWord['word']);
-                        if (strpos($sNormQuery, $sNormPostcode) !== false) {
-                            $oValidTokens->addToken(
-                                $sTok,
-                                new Token\Postcode($iId, $aWord['word'], null)
-                            );
+                        $iSplitPos = strpos($aWord['word'], '@');
+                        if ($iSplitPos === false) {
+                            $sPostcode = $aWord['word'];
+                        } else {
+                            $sPostcode = substr($aWord['word'], 0, $iSplitPos);
                         }
+
+                        $oValidTokens->addToken(
+                            $sTok,
+                            new Token\Postcode($iId, $sPostcode, null)
+                        );
                     }
                     break;
                 case 'S':  // tokens for classification terms (special phrases)

lib-php/tokenizer/legacy_tokenizer.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 namespace Nominatim;
 

lib-php/website/deletable.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 require_once(CONST_LibDir.'/init-website.php');
 require_once(CONST_LibDir.'/log.php');

lib-php/website/details.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 require_once(CONST_LibDir.'/init-website.php');
 require_once(CONST_LibDir.'/log.php');
@@ -198,7 +206,7 @@ if ($bIncludeLinkedPlaces) {
     $aLinkedLines = $oDB->getAll($sSQL);
 }
 
-// All places this is an imediate parent of
+// All places this is an immediate parent of
 $aHierarchyLines = false;
 if ($bIncludeHierarchy) {
     $sSQL = 'SELECT obj.place_id, osm_type, osm_id, class, type, housenumber,';

lib-php/website/lookup.php

@@ -1,4 +1,12 @@
 <?php
+/**
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This file is part of Nominatim. (https://nominatim.org)
+ *
+ * Copyright (C) 2022 by the Nominatim developer community.
+ * For a full list of authors see the git log.
+ */
 
 require_once(CONST_LibDir.'/init-website.php');
 require_once(CONST_LibDir.'/log.php');