prepare 3.7.3 release

Fixes #2350.

.github/actions/build-nominatim/action.yml

@@ -1,26 +1,13 @@
 name: 'Build Nominatim'
 
-inputs:
-    ubuntu:
-        description: 'Version of Ubuntu to install on'
-        required: false
-        default: '20'
-
 runs:
     using: "composite"
 
     steps:
         - name: Install prerequisites
           run: |
-            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev
-            if [ "x$UBUNTUVER" == "x18" ]; then
-                pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium PyYAML==5.1 datrie
-            else
-                sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
-            fi
+            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil python3-jinja2 python3-icu
           shell: bash
-          env:
-            UBUNTUVER: ${{ inputs.ubuntu }}
 
         - name: Download dependencies
           run: |

.github/workflows/ci-tests.yml

@@ -4,22 +4,16 @@ on: [ push, pull_request ]
 
 jobs:
     tests:
+        runs-on: ubuntu-20.04
+
         strategy:
             matrix:
-                ubuntu: [18, 20]
+                postgresql: [9.5, 13]
                 include:
-                    - ubuntu: 18
-                      postgresql: 9.5
+                    - postgresql: 9.5
                       postgis: 2.5
-                      pytest: pytest
-                      php: 7.2
-                    - ubuntu: 20
-                      postgresql: 13
+                    - postgresql: 13
                       postgis: 3
-                      pytest: py.test-3
-                      php: 7.4
-
-        runs-on: ubuntu-${{ matrix.ubuntu }}.04
 
         steps:
             - uses: actions/checkout@v2
@@ -30,14 +24,8 @@ jobs:
             - name: Setup PHP
               uses: shivammathur/setup-php@v2
               with:
-                  php-version: ${{ matrix.php }}
-                  coverage: xdebug
-                  tools: phpunit, phpcs, composer
-
-            - uses: actions/setup-python@v2
-              with:
-                python-version: 3.6
-              if: matrix.ubuntu == 18
+                  php-version: '7.4'
+                  tools: phpunit, phpcs
 
             - name: Get Date
               id: get-date
@@ -55,69 +43,30 @@ jobs:
               with:
                   postgresql-version: ${{ matrix.postgresql }}
                   postgis-version: ${{ matrix.postgis }}
-
             - uses: ./Nominatim/.github/actions/build-nominatim
-              with:
-                  ubuntu: ${{ matrix.ubuntu }}
 
             - name: Install test prerequsites
-              run: sudo apt-get install -y -qq pylint python3-pytest python3-behave python3-pytest-cov php-codecoverage
-              if: matrix.ubuntu == 20
-
-            - name: Install test prerequsites
-              run: |
-                   pip3 install pylint==2.6.0 pytest pytest-cov behave==1.2.6
-              if: matrix.ubuntu == 18
+              run: sudo apt-get install -y -qq php-codesniffer pylint python3-pytest python3-behave
 
             - name: PHP linting
               run: phpcs --report-width=120 .
               working-directory: Nominatim
 
             - name: Python linting
-              run: pylint nominatim
+              run: pylint --extension-pkg-whitelist=osmium nominatim
               working-directory: Nominatim
 
             - name: PHP unit tests
-              run: phpunit --coverage-clover ../../coverage-php.xml ./
+              run: phpunit ./
               working-directory: Nominatim/test/php
-              if: matrix.ubuntu == 20
 
             - name: Python unit tests
-              run: $PYTEST --cov=nominatim --cov-report=xml test/python
+              run: py.test-3 test/python
               working-directory: Nominatim
-              env:
-                PYTEST: ${{ matrix.pytest }}
 
             - name: BDD tests
-              run: |
-                  mkdir cov
-                  behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3 -DPHPCOV=./cov
-                  composer require phpunit/phpcov:7.0.2
-                  vendor/bin/phpcov merge --clover ../../coverage-bdd.xml ./cov
+              run: behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
               working-directory: Nominatim/test/bdd
-              if: matrix.ubuntu == 20
-
-            - name: BDD tests
-              run: |
-                  behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build --format=progress3
-              working-directory: Nominatim/test/bdd
-              if: matrix.ubuntu == 18
-
-            - name: BDD tests (legacy_icu tokenizer)
-              run: |
-                  behave -DREMOVE_TEMPLATE=1 -DBUILDDIR=$GITHUB_WORKSPACE/build -DTOKENIZER=legacy_icu --format=progress3
-              working-directory: Nominatim/test/bdd
-
-            - name: Upload coverage to Codecov
-              uses: codecov/codecov-action@v1
-              with:
-                files: ./Nominatim/coverage*.xml
-                directory: ./
-                name: codecov-umbrella
-                fail_ci_if_error: false
-                path_to_write_report: ./coverage/codecov_report.txt
-                verbose: true
-              if: matrix.ubuntu == 20
 
     import:
         strategy:
@@ -159,7 +108,7 @@ jobs:
 
             - uses: actions/setup-python@v2
               with:
-                python-version: 3.6
+                python-version: 3.5
               if: matrix.ubuntu == 18
 
             - uses: ./Nominatim/.github/actions/setup-postgresql
@@ -167,8 +116,13 @@ jobs:
                   postgresql-version: ${{ matrix.postgresql }}
                   postgis-version: ${{ matrix.postgis }}
             - uses: ./Nominatim/.github/actions/build-nominatim
-              with:
-                  ubuntu: ${{ matrix.ubuntu }}
+
+            - name: Install extra dependencies for Ubuntu 18
+              run: |
+                sudo apt-get install libicu-dev
+                pip3 install python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu osmium
+              shell: bash
+              if: matrix.ubuntu == 18
 
             - name: Clean installation
               run: rm -rf Nominatim build

.pylintrc

@@ -1,7 +1,7 @@
 [MASTER]
 
 extension-pkg-whitelist=osmium
-ignored-modules=icu,datrie
+ignored-modules=icu
 
 [MESSAGES CONTROL]
 
@@ -10,6 +10,3 @@ ignored-modules=icu,datrie
 # closing added here because it sometimes triggers a false positive with
 # 'with' statements.
 ignored-classes=NominatimArgs,closing
-disable=too-few-public-methods,duplicate-code
-
-good-names=i,x,y,fd,db

CMakeLists.txt

@@ -20,7 +20,7 @@ project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 3)
 set(NOMINATIM_VERSION_MINOR 7)
-set(NOMINATIM_VERSION_PATCH 0)
+set(NOMINATIM_VERSION_PATCH 3)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 
@@ -62,7 +62,7 @@ endif()
 #-----------------------------------------------------------------------------
 
 if (BUILD_IMPORTER)
-    find_package(PythonInterp 3.6 REQUIRED)
+    find_package(PythonInterp 3.5 REQUIRED)
 endif()
 
 #-----------------------------------------------------------------------------
@@ -109,6 +109,21 @@ if (BUILD_IMPORTER)
                            "    wget -O ${PROJECT_SOURCE_DIR}/data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz")
    endif()
 
+   set(CUSTOMSCRIPTS
+       check_import_finished.php
+       country_languages.php
+       export.php
+       query.php
+       setup.php
+       update.php
+       warm.php
+      )
+
+   foreach (script_source ${CUSTOMSCRIPTS})
+       configure_file(${PROJECT_SOURCE_DIR}/cmake/script.tmpl
+                      ${PROJECT_BINARY_DIR}/utils/${script_source})
+   endforeach()
+
    configure_file(${PROJECT_SOURCE_DIR}/cmake/tool.tmpl
                   ${PROJECT_BINARY_DIR}/nominatim)
 endif()
@@ -258,6 +273,4 @@ install(FILES settings/env.defaults
               settings/import-address.style
               settings/import-full.style
               settings/import-extratags.style
-              settings/legacy_icu_tokenizer.yaml
-              settings/icu-rules/extended-unicode-to-asccii.yaml
         DESTINATION ${NOMINATIM_CONFIGDIR})

ChangeLog

@@ -1,3 +1,23 @@
+3.7.3
+
+ * fix XSS vulnerability in debug view
+
+3.7.2
+
+ * fix database check for reverse-only imports
+ * do not error out in status API result when import date is missing
+ * add array_key_last function for PHP < 7.3 (thanks to @woodpack)
+ * fix more url when server name is unknown (thanks to @mogita)
+ * commit changes to replication log table
+
+3.7.1
+
+ * fix smaller issues with special phrases import
+ * add index to speed up continued indexing during import
+ * fix index on location_property_tiger(parent_place_id)
+ * make sure Python code is backward-compatible with Python 3.5
+ * various documentation fixes
+
 3.7.0
 
  * switch to dotenv for configuration file

README.md

@@ -1,5 +1,4 @@
 [![Build Status](https://github.com/osm-search/Nominatim/workflows/CI%20Tests/badge.svg)](https://github.com/osm-search/Nominatim/actions?query=workflow%3A%22CI+Tests%22)
-[![codecov](https://codecov.io/gh/osm-search/Nominatim/branch/master/graph/badge.svg?token=8P1LXrhCMy)](https://codecov.io/gh/osm-search/Nominatim)
 
 Nominatim
 =========

SECURITY.md

@@ -1,39 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-All Nominatim releases receive security updates for two years.
-
-The following table lists the end of support for all currently supported
-versions.
-
-| Version | End of support for security updates |
-| ------- | ----------------------------------- |
-| 3.7.x   | 2023-04-05                          |
-| 3.6.x   | 2022-12-12                          |
-| 3.5.x   | 2022-06-05                          |
-| 3.4.x   | 2021-10-24                          |
-
-## Reporting a Vulnerability
-
-If you believe, you have found an issue in Nominatim that has implications on
-security, please send a description of the issue to **security@nominatim.org**.
-You will receive an acknowledgement of your mail within 3 work days where we
-also notify you of the next steps.
-
-## How we Disclose Security Issues
-
-** The following section only applies to security issues found in released
-versions. Issues that concern the master development branch only will be
-fixed immediately on the branch with the corresponding PR containing the
-description of the nature and severity of the issue. **
-
-Patches for identified security issues are applied to all affected versions and
-new minor versions are released. At the same time we release a statement at
-the [Nominatim blog](https://nominatim.org/blog/) describing the nature of the
-incident. Announcements will also be published at the
-[geocoding mailinglist](https://lists.openstreetmap.org/listinfo/geocoding).
-
-## List of Previous Incidents
-
-* 2020-05-04 - [SQL injection issue on /details endpoint](https://lists.openstreetmap.org/pipermail/geocoding/2020-May/002012.html)

cmake/script.tmpl

@@ -0,0 +1,14 @@
+#!@PHP_BIN@ -Cq
+<?php
+require('@CMAKE_SOURCE_DIR@/lib-php/dotenv_loader.php');
+
+@define('CONST_Default_ModulePath', '@CMAKE_BINARY_DIR@/module');
+@define('CONST_Default_Osm2pgsql', '@CMAKE_BINARY_DIR@/osm2pgsql/osm2pgsql');
+@define('CONST_DataDir', '@CMAKE_SOURCE_DIR@/data');
+@define('CONST_SqlDir', '@CMAKE_SOURCE_DIR@/lib-sql');
+@define('CONST_ConfigDir', '@CMAKE_SOURCE_DIR@/settings');
+
+loadDotEnv();
+$_SERVER['NOMINATIM_NOMINATIM_TOOL'] = '@CMAKE_BINARY_DIR@/nominatim';
+
+require_once('@CMAKE_SOURCE_DIR@/lib-php/admin/@script_source@');

codecov.yml

@@ -1,14 +0,0 @@
-codecov:
-  require_ci_to_pass: yes
-
-coverage:
-  status:
-    project: off
-    patch: off
-
-comment:
-  require_changes: true
-  after_n_builds: 2
-
-fixes:
-  - "Nominatim/::"

data/words.sql

@@ -29787,7 +29787,7 @@ st	5557484
 
 -- prefill word table
 
-select count(precompute_words(v)) from (select distinct svals(name) as v from place) as w where v is not null;
+select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;
 select count(getorcreate_housenumber_id(make_standard_name(v))) from (select distinct address->'housenumber' as v from place where address ? 'housenumber') as w;
 
 -- copy the word frequencies

docs/admin/Customization.md

@@ -1,101 +0,0 @@
-# Customization of the Database
-
-This section explains in detail how to configure a Nominatim import and
-the various means to use external data.
-
-## External postcode data
-
-Nominatim creates a table of known postcode centroids during import. This table
-is used for searches of postcodes and for adding postcodes to places where the
-OSM data does not provide one. These postcode centroids are mainly computed
-from the OSM data itself. In addition, Nominatim supports reading postcode
-information from an external CSV file, to supplement the postcodes that are
-missing in OSM.
-
-To enable external postcode support, simply put one CSV file per country into
-your project directory and name it `<CC>_postcodes.csv`. `<CC>` must be the
-two-letter country code for which to apply the file. The file may also be
-gzipped. Then it must be called `<CC>_postcodes.csv.gz`.
-
-The CSV file must use commas as a delimiter and have a header line. Nominatim
-expects three columns to be present: `postcode`, `lat` and `lon`. All other
-columns are ignored. `lon` and `lat` must describe the x and y coordinates of the
-postcode centroids in WGS84.
-
-The postcode files are loaded only when there is data for the given country
-in your database. For example, if there is a `us_postcodes.csv` file in your
-project directory but you import only an excerpt of Italy, then the US postcodes
-will simply be ignored.
-
-As a rule, the external postcode data should be put into the project directory
-**before** starting the initial import. Still, you can add, remove and update the
-external postcode data at any time. Simply
-run:
-
-```
-nominatim refresh --postcodes
-```
-
-to make the changes visible in your database. Be aware, however, that the changes
-only have an immediate effect on searches for postcodes. Postcodes that were
-added to places are only updated, when they are reindexed. That usually happens
-only during replication updates.
-
-## Installing Tiger housenumber data for the US
-
-Nominatim is able to use the official [TIGER](https://www.census.gov/geographies/mapping-files/time-series/geo/tiger-line-file.html)
-address set to complement the OSM house number data in the US. You can add
-TIGER data to your own Nominatim instance by following these steps. The
-entire US adds about 10GB to your database.
-
-  1. Get preprocessed TIGER 2020 data:
-
-        cd $PROJECT_DIR
-        wget https://nominatim.org/data/tiger2020-nominatim-preprocessed.csv.tar.gz
-
-  2. Import the data into your Nominatim database:
-
-        nominatim add-data --tiger-data tiger2020-nominatim-preprocessed.csv.tar.gz
-
-  3. Enable use of the Tiger data in your `.env` by adding:
-
-        echo NOMINATIM_USE_US_TIGER_DATA=yes >> .env
-
-  4. Apply the new settings:
-
-        nominatim refresh --functions
-
-
-See the [developer's guide](../develop/data-sources.md#us-census-tiger) for more
-information on how the data got preprocessed.
-
-## Special phrases import
-
-As described in the [Importation chapter](Import.md), it is possible to
-import special phrases from the wiki with the following command:
-
-```sh
-nominatim special-phrases --import-from-wiki
-```
-
-But, it is also possible to import some phrases from a csv file. 
-To do so, you have access to the following command:
-
-```sh
-nominatim special-phrases --import-from-csv <csv file>
-```
-
-Note that the two previous import commands will update the phrases from your database.
-This means that if you import some phrases from a csv file, only the phrases
-present in the csv file will be kept into the database. All other phrases will
-be removed.
-
-If you want to only add new phrases and not update the other ones you can add
-the argument `--no-replace` to the import command. For example:
-
-```sh
-nominatim special-phrases --import-from-csv <csv file> --no-replace
-```
-
-This will add the phrases present in the csv file into the database without
-removing the other ones.

docs/admin/Import.md

@@ -40,7 +40,7 @@ all commands from the project directory.
 
 ### Configuration setup in `.env`
 
-The Nominatim server can be customized via an `.env` configuration file in the 
+The Nominatim server can be customized via an `.env` configuration file in the
 project directory. This is a file in [dotenv](https://github.com/theskumar/python-dotenv)
 format which looks the same as variable settings in a standard shell environment.
 You can also set the same configuration via environment variables. All
@@ -48,7 +48,7 @@ settings have a `NOMINATIM_` prefix to avoid conflicts with other environment
 variables.
 
 There are lots of configuration settings you can tweak. Have a look
-at `Nominatim/settings/env.default` for a full list. Most should have a sensible default.
+at `settings/env.default` for a full list. Most should have a sensible default.
 
 #### Flatnode files
 
@@ -83,19 +83,15 @@ The file is about 400MB and adds around 4GB to the Nominatim database.
     `nominatim refresh --wiki-data --importance`. Updating importances for
     a planet can take a couple of hours.
 
-### External postcodes
+### Great Britain, USA postcodes
 
-Nominatim can use postcodes from an external source to improve searching with
-postcodes. We provide precomputed postcodes sets for the US (using TIGER data)
-and the UK (using the [CodePoint OpenData set](https://osdatahub.os.uk/downloads/open/CodePointOpen).
-This data can be optionally downloaded into the project directory:
+Nominatim can use postcodes from an external source to improve searches that
+involve a GB or US postcode. This data can be optionally downloaded into the
+project directory:
 
     cd $PROJECT_DIR
-    wget https://www.nominatim.org/data/gb_postcodes.csv.gz
-    wget https://www.nominatim.org/data/us_postcodes.csv.gz
-
-You can also add your own custom postcode sources, see
-[Customization of postcodes](Customization.md#external-postcode-data).
+    wget https://www.nominatim.org/data/gb_postcode_data.sql.gz
+    wget https://www.nominatim.org/data/us_postcode_data.sql.gz
 
 ## Choosing the data to import
 
@@ -193,15 +189,12 @@ can be found in the development section.
     [Geofabrik](https://download.geofabrik.de).
 
 Download the data to import. Then issue the following command
-from the **project directory** to start the import:
+from the **build directory** to start the import:
 
 ```sh
 nominatim import --osm-file <data file> 2>&1 | tee setup.log
 ```
 
-The **project directory** is the one that you have set up at the beginning.
-See [creating the project directory](Import#creating-the-project-directory).
-
 ### Notes on full planet imports
 
 Even on a perfectly configured machine
@@ -255,9 +248,6 @@ to verify that your installation is working. Go to
 `http://localhost:8088/status.php` and you should see the message `OK`.
 You can also run a search query, e.g. `http://localhost:8088/search.php?q=Berlin`.
 
-Note that search query is not supported for reverse-only imports. You can run a
-reverse query, e.g. `http://localhost:8088/reverse.php?lat=27.1750090510034&lon=78.04209025`.
-
 To run Nominatim via webservers like Apache or nginx, please read the
 [Deployment chapter](Deployment.md).
 
@@ -280,12 +270,37 @@ If you want to be able to search for places by their type through
 [special key phrases](https://wiki.openstreetmap.org/wiki/Nominatim/Special_Phrases)
 you also need to import these key phrases like this:
 
-```sh
-nominatim special-phrases --import-from-wiki
-```
+    nominatim special-phrases --import-from-wiki
 
 Note that this command downloads the phrases from the wiki link above. You
 need internet access for the step.
 
-You can also import special phrases from a csv file, for more 
-information please read the [Customization chapter](Customization.md).
+
+## Installing Tiger housenumber data for the US
+
+Nominatim is able to use the official [TIGER](https://www.census.gov/geographies/mapping-files/time-series/geo/tiger-line-file.html)
+address set to complement the OSM house number data in the US. You can add
+TIGER data to your own Nominatim instance by following these steps. The
+entire US adds about 10GB to your database.
+
+  1. Get preprocessed TIGER 2020 data:
+
+        cd $PROJECT_DIR
+        wget https://nominatim.org/data/tiger2020-nominatim-preprocessed.tar.gz
+
+  2. Import the data into your Nominatim database:
+
+        nominatim add-data --tiger-data tiger2020-nominatim-preprocessed.tar.gz
+
+  3. Enable use of the Tiger data in your `.env` by adding:
+
+        echo NOMINATIM_USE_US_TIGER_DATA=yes >> .env
+
+  4. Apply the new settings:
+
+        nominatim refresh --functions
+
+
+See the [developer's guide](../develop/data-sources.md#us-census-tiger) for more
+information on how the data got preprocessed.
+

docs/admin/Installation.md

@@ -17,7 +17,6 @@ and can't offer support.
 
   * [Docker](https://github.com/mediagis/nominatim-docker)
   * [Docker on Kubernetes](https://github.com/peter-evans/nominatim-k8s)
-  * [Kubernetes with Helm](https://github.com/robjuz/helm-charts/blob/master/charts/nominatim/README.md)
   * [Ansible](https://github.com/synthesio/infra-ansible-nominatim)
 
 ## Prerequisites
@@ -38,16 +37,14 @@ For compiling:
 
 For running Nominatim:
 
-  * [PostgreSQL](https://www.postgresql.org) (9.5+ will work, 11+ strongly recommended)
+  * [PostgreSQL](https://www.postgresql.org) (9.3+ will work, 11+ strongly recommended)
   * [PostGIS](https://postgis.net) (2.2+)
-  * [Python 3](https://www.python.org/) (3.6+)
+  * [Python 3](https://www.python.org/) (3.5+)
   * [Psycopg2](https://www.psycopg.org) (2.7+)
   * [Python Dotenv](https://github.com/theskumar/python-dotenv)
   * [psutil](https://github.com/giampaolo/psutil)
   * [Jinja2](https://palletsprojects.com/p/jinja/)
   * [PyICU](https://pypi.org/project/PyICU/)
-  * [PyYaml](https://pyyaml.org/) (5.1+)
-  * [datrie](https://github.com/pytries/datrie)
   * [PHP](https://php.net) (7.0 or later)
   * PHP-pgsql
   * PHP-intl (bundled with PHP)

docs/admin/Migration.md

@@ -17,17 +17,6 @@ breaking changes. **Please read them before running the migration.**
 
 ## 3.6.0 -> 3.7.0
 
-### New format and name of configuration file
-
-The configuration for an import is now saved in a `.env` file in the project
-directory. This file follows the dotenv format. For more information, see
-the [installation chapter](Import.md#configuration-setup-in-env).
-
-To migrate to the new system, create a new project directory, add the `.env`
-file and port your custom configuration from `settings/local.php`. Most
-settings are named similar and only have received a `NOMINATIM_` prefix.
-Use the default settings in `settings/env.defaults` as a reference.
-
 ### New location for data files
 
 External data files for Wikipedia importance, postcodes etc. are no longer
@@ -124,14 +113,6 @@ configuration file, run the following command after updating:
 ./utils/setup.php --setup-website
 ```
 
-### Update SQL code
-
-To update the SQL code to the leatest version run:
-
-```
-./utils/setup.php --create-functions --enable-diff-updates --create-partition-functions
-```
-
 ## 3.4.0 -> 3.5.0
 
 ### New Wikipedia/Wikidata importance tables

docs/admin/Setup-Nominatim-UI.md

@@ -10,11 +10,12 @@ installation. For more details, please also have a look at the
 
 ## Installing nominatim-ui
 
-We provide regular releases of nominatim-ui that contain the packaged website.
-They do not need any special installation. Just download, configure
-and run it. Grab the latest release from
-[nominatim-ui's Github release page](https://github.com/osm-search/nominatim-ui/releases)
-and unpack it. You can use `nominatim-ui-x.x.x.tar.gz` or `nominatim-ui-x.x.x.zip`.
+nominatim-ui does not need any special installation, just download, configure
+and run it.
+
+Clone the source from github:
+
+    git clone https://github.com/osm-search/nominatim-ui
 
 Copy the example configuration into the right place:
 

docs/admin/Tokenizers.md

@@ -1,205 +0,0 @@
-# Tokenizers
-
-The tokenizer module in Nominatim is responsible for analysing the names given
-to OSM objects and the terms of an incoming query in order to make sure, they
-can be matched appropriately.
-
-Nominatim offers different tokenizer modules, which behave differently and have
-different configuration options. This sections describes the tokenizers and how
-they can be configured.
-
-!!! important
-    The use of a tokenizer is tied to a database installation. You need to choose
-    and configure the tokenizer before starting the initial import. Once the import
-    is done, you cannot switch to another tokenizer anymore. Reconfiguring the
-    chosen tokenizer is very limited as well. See the comments in each tokenizer
-    section.
-
-## Legacy tokenizer
-
-The legacy tokenizer implements the analysis algorithms of older Nominatim
-versions. It uses a special Postgresql module to normalize names and queries.
-This tokenizer is currently the default.
-
-To enable the tokenizer add the following line to your project configuration:
-
-```
-NOMINATIM_TOKENIZER=legacy
-```
-
-The Postgresql module for the tokenizer is available in the `module` directory
-and also installed with the remainder of the software under
-`lib/nominatim/module/nominatim.so`. You can specify a custom location for
-the module with
-
-```
-NOMINATIM_DATABASE_MODULE_PATH=<path to directory where nominatim.so resides>
-```
-
-This is in particular useful when the database runs on a different server.
-See [Advanced installations](Advanced-Installations.md#importing-nominatim-to-an-external-postgresql-database) for details.
-
-There are no other configuration options for the legacy tokenizer. All
-normalization functions are hard-coded.
-
-## ICU tokenizer
-
-!!! danger
-    This tokenizer is currently in active development and still subject
-    to backwards-incompatible changes.
-
-The ICU tokenizer uses the [ICU library](http://site.icu-project.org/) to
-normalize names and queries. It also offers configurable decomposition and
-abbreviation handling.
-
-### How it works
-
-On import the tokenizer processes names in the following four stages:
-
-1. The **Normalization** part removes all non-relevant information from the
-   input.
-2. Incoming names are now converted to **full names**. This process is currently
-   hard coded and mostly serves to handle name tags from OSM that contain
-   multiple names (e.g. [Biel/Bienne](https://www.openstreetmap.org/node/240097197)).
-3. Next the tokenizer creates **variants** from the full names. These variants
-   cover decomposition and abbreviation handling. Variants are saved to the
-   database, so that it is not necessary to create the variants for a search
-   query.
-4. The final **Tokenization** step converts the names to a simple ASCII form,
-   potentially removing further spelling variants for better matching.
-
-At query time only stage 1) and 4) are used. The query is normalized and
-tokenized and the resulting string used for searching in the database.
-
-### Configuration
-
-The ICU tokenizer is configured using a YAML file which can be configured using
-`NOMINATIM_TOKENIZER_CONFIG`. The configuration is read on import and then
-saved as part of the internal database status. Later changes to the variable
-have no effect.
-
-Here is an example configuration file:
-
-``` yaml
-normalization:
-    - ":: lower ()"
-    - "ß > 'ss'" # German szet is unimbigiously equal to double ss
-transliteration:
-    - !include /etc/nominatim/icu-rules/extended-unicode-to-asccii.yaml
-    - ":: Ascii ()"
-variants:
-    - language: de
-      words:
-        - ~haus => haus
-        - ~strasse -> str
-    - language: en
-      words: 
-        - road -> rd
-        - bridge -> bdge,br,brdg,bri,brg
-```
-
-The configuration file contains three sections:
-`normalization`, `transliteration`, `variants`.
-
-The normalization and transliteration sections each must contain a list of
-[ICU transformation rules](https://unicode-org.github.io/icu/userguide/transforms/general/rules.html).
-The rules are applied in the order in which they appear in the file.
-You can also include additional rules from external yaml file using the
-`!include` tag. The included file must contain a valid YAML list of ICU rules
-and may again include other files.
-
-!!! warning
-    The ICU rule syntax contains special characters that conflict with the
-    YAML syntax. You should therefore always enclose the ICU rules in
-    double-quotes.
-
-The variants section defines lists of replacements which create alternative
-spellings of a name. To create the variants, a name is scanned from left to
-right and the longest matching replacement is applied until the end of the
-string is reached.
-
-The variants section must contain a list of replacement groups. Each group
-defines a set of properties that describes where the replacements are
-applicable. In addition, the word section defines the list of replacements
-to be made. The basic replacement description is of the form:
-
-```
-<source>[,<source>[...]] => <target>[,<target>[...]]
-```
-
-The left side contains one or more `source` terms to be replaced. The right side
-lists one or more replacements. Each source is replaced with each replacement
-term.
-
-!!! tip
-    The source and target terms are internally normalized using the
-    normalization rules given in the configuration. This ensures that the
-    strings match as expected. In fact, it is better to use unnormalized
-    words in the configuration because then it is possible to change the
-    rules for normalization later without having to adapt the variant rules.
-
-#### Decomposition
-
-In its standard form, only full words match against the source. There
-is a special notation to match the prefix and suffix of a word:
-
-``` yaml
-- ~strasse => str  # matches "strasse" as full word and in suffix position
-- hinter~ => hntr  # matches "hinter" as full word and in prefix position
-```
-
-There is no facility to match a string in the middle of the word. The suffix
-and prefix notation automatically trigger the decomposition mode: two variants
-are created for each replacement, one with the replacement attached to the word
-and one separate. So in above example, the tokenization of "hauptstrasse" will
-create the variants "hauptstr" and "haupt str". Similarly, the name "rote strasse"
-triggers the variants "rote str" and "rotestr". By having decomposition work
-both ways, it is sufficient to create the variants at index time. The variant
-rules are not applied at query time.
-
-To avoid automatic decomposition, use the '|' notation:
-
-``` yaml
-- ~strasse |=> str
-```
-
-simply changes "hauptstrasse" to "hauptstr" and "rote strasse" to "rote str".
-
-#### Initial and final terms
-
-It is also possible to restrict replacements to the beginning and end of a
-name:
-
-``` yaml
-- ^south => s  # matches only at the beginning of the name
-- road$ => rd  # matches only at the end of the name
-```
-
-So the first example would trigger a replacement for "south 45th street" but
-not for "the south beach restaurant".
-
-#### Replacements vs. variants
-
-The replacement syntax `source => target` works as a pure replacement. It changes
-the name instead of creating a variant. To create an additional version, you'd
-have to write `source => source,target`. As this is a frequent case, there is
-a shortcut notation for it:
-
-```
-<source>[,<source>[...]] -> <target>[,<target>[...]]
-```
-
-The simple arrow causes an additional variant to be added. Note that
-decomposition has an effect here on the source as well. So a rule
-
-``` yaml
-- "~strasse -> str"
-```
-
-means that for a word like `hauptstrasse` four variants are created:
-`hauptstrasse`, `haupt strasse`, `hauptstr` and `haupt str`.
-
-### Reconfiguration
-
-Changing the configuration after the import is currently not possible, although
-this feature may be added at a later time.

docs/admin/Update.md

@@ -30,9 +30,9 @@ diffs for Ireland from Geofabrik add the following:
 
     # base URL of the replication service
     NOMINATIM_REPLICATION_URL="https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates"
-    # How often upstream publishes diffs (in seconds)
+    # How often upstream publishes diffs
     NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400
-    # How long to sleep if no update found yet (in seconds)
+    # How long to sleep if no update found yet
     NOMINATIM_REPLICATION_RECHECK_INTERVAL=900
 
 To set up the update process now run the following command:

docs/develop/Development-Environment.md

@@ -29,7 +29,7 @@ The Nominatim test suite consists of behavioural tests (using behave) and
 unit tests (using PHPUnit for PHP code and pytest for Python code).
 It has the following additional requirements:
 
-* [behave test framework](https://behave.readthedocs.io) >= 1.2.6
+* [behave test framework](https://behave.readthedocs.io) >= 1.2.5
 * [phpunit](https://phpunit.de) >= 7.3
 * [PHP CodeSniffer](https://github.com/squizlabs/PHP_CodeSniffer)
 * [Pylint](https://pylint.org/) (2.6.0 is used for the CI)

docs/mkdocs.yml

@@ -19,8 +19,6 @@ pages:
         - 'Import' : 'admin/Import.md'
         - 'Update' : 'admin/Update.md'
         - 'Deploy' : 'admin/Deployment.md'
-        - 'Customize Imports' : 'admin/Customization.md'
-        - 'Tokenizers' : 'admin/Tokenizers.md'
         - 'Nominatim UI'  : 'admin/Setup-Nominatim-UI.md'
         - 'Advanced Installations' : 'admin/Advanced-Installations.md'
         - 'Migration from older Versions' : 'admin/Migration.md'

lib-php/AddressDetails.php

@@ -61,7 +61,7 @@ class AddressDetails
         return join(', ', $aParts);
     }
 
-    public function getAddressNames()
+    public function getAddressNames($sCountry = null)
     {
         $aAddress = array();
 
@@ -79,11 +79,13 @@ class AddressDetails
                 $sName = $aLine['housenumber'];
             }
 
-            if (isset($sName)
-                && (!isset($aAddress[$sTypeLabel])
-                    || $aLine['class'] == 'place')
-            ) {
-                $aAddress[$sTypeLabel] = $sName;
+            if (isset($sName)) {
+                $sTypeLabel = strtolower(str_replace(' ', '_', $sTypeLabel));
+                if (!isset($aAddress[$sTypeLabel])
+                    || $aLine['class'] == 'place'
+                ) {
+                    $aAddress[$sTypeLabel] = $sName;
+                }
             }
         }
 

lib-php/DB.php

@@ -39,9 +39,7 @@ class DB
         $conn->exec("SET DateStyle TO 'sql,european'");
         $conn->exec("SET client_encoding TO 'utf-8'");
         $iMaxExecution = ini_get('max_execution_time');
-        if ($iMaxExecution > 0) {
-            $conn->setAttribute(\PDO::ATTR_TIMEOUT, $iMaxExecution); // seconds
-        }
+        if ($iMaxExecution > 0) $conn->setAttribute(\PDO::ATTR_TIMEOUT, $iMaxExecution); // seconds
 
         $this->connection = $conn;
         return true;
@@ -97,9 +95,7 @@ class DB
         try {
             $stmt = $this->getQueryStatement($sSQL, $aInputVars, $sErrMessage);
             $row = $stmt->fetch(\PDO::FETCH_NUM);
-            if ($row === false) {
-                return false;
-            }
+            if ($row === false) return false;
         } catch (\PDOException $e) {
             throw new \Nominatim\DatabaseError($sErrMessage, 500, null, $e, $sSQL);
         }
@@ -310,13 +306,9 @@ class DB
         if (preg_match('/^pgsql:(.+)$/', $sDSN, $aMatches)) {
             foreach (explode(';', $aMatches[1]) as $sKeyVal) {
                 list($sKey, $sVal) = explode('=', $sKeyVal, 2);
-                if ($sKey == 'host') {
-                    $sKey = 'hostspec';
-                } elseif ($sKey == 'dbname') {
-                    $sKey = 'database';
-                } elseif ($sKey == 'user') {
-                    $sKey = 'username';
-                }
+                if ($sKey == 'host') $sKey = 'hostspec';
+                if ($sKey == 'dbname') $sKey = 'database';
+                if ($sKey == 'user') $sKey = 'username';
                 $aInfo[$sKey] = $sVal;
             }
         }

lib-php/DatabaseError.php

@@ -5,7 +5,7 @@ namespace Nominatim;
 class DatabaseError extends \Exception
 {
 
-    public function __construct($message, $code, $previous, $oPDOErr, $sSql = null)
+    public function __construct($message, $code = 500, Exception $previous = null, $oPDOErr, $sSql = null)
     {
         parent::__construct($message, $code, $previous);
         // https://secure.php.net/manual/en/class.pdoexception.php

lib-php/DebugHtml.php

@@ -78,7 +78,7 @@ class Debug
         echo '<th>Address Tokens</th><th>Address Not</th>';
         echo '<th>country</th><th>operator</th>';
         echo '<th>class</th><th>type</th><th>postcode</th><th>housenumber</th></tr>';
-        foreach ($aSearches as $aRankedSet) {
+        foreach ($aSearches as $iRank => $aRankedSet) {
             foreach ($aRankedSet as $aRow) {
                 $aRow->dumpAsHtmlTableRow($aWordsIDs);
             }
@@ -127,7 +127,7 @@ class Debug
 
     public static function printSQL($sSQL)
     {
-        echo '<p><tt><font color="#aaa">'.$sSQL.'</font></tt></p>'."\n";
+        echo '<p><tt><font color="#aaa">'.htmlspecialchars($sSQL, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401).'</font></tt></p>'."\n";
     }
 
     private static function outputVar($mVar, $sPreNL)
@@ -170,11 +170,12 @@ class Debug
         }
 
         if (is_string($mVar)) {
-            echo "'$mVar'";
-            return strlen($mVar) + 2;
+            $sOut = "'$mVar'";
+        } else {
+            $sOut = (string)$mVar;
         }
 
-        echo (string)$mVar;
-        return strlen((string)$mVar);
+        echo htmlspecialchars($sOut, ENT_QUOTES | ENT_SUBSTITUTE | ENT_HTML401);
+        return strlen($sOut);
     }
 }

lib-php/Geocode.php

@@ -7,20 +7,18 @@ require_once(CONST_LibDir.'/Phrase.php');
 require_once(CONST_LibDir.'/ReverseGeocode.php');
 require_once(CONST_LibDir.'/SearchDescription.php');
 require_once(CONST_LibDir.'/SearchContext.php');
-require_once(CONST_LibDir.'/SearchPosition.php');
 require_once(CONST_LibDir.'/TokenList.php');
-require_once(CONST_TokenizerDir.'/tokenizer.php');
 
 class Geocode
 {
     protected $oDB;
 
     protected $oPlaceLookup;
-    protected $oTokenizer;
 
     protected $aLangPrefOrder = array();
 
     protected $aExcludePlaceIDs = array();
+    protected $bReverseInPlan = true;
 
     protected $iLimit = 20;
     protected $iFinalLimit = 10;
@@ -44,12 +42,28 @@ class Geocode
     protected $sQuery = false;
     protected $aStructuredQuery = false;
 
+    protected $oNormalizer = null;
+
 
     public function __construct(&$oDB)
     {
         $this->oDB =& $oDB;
         $this->oPlaceLookup = new PlaceLookup($this->oDB);
-        $this->oTokenizer = new \Nominatim\Tokenizer($this->oDB);
+        $this->oNormalizer = \Transliterator::createFromRules(CONST_Term_Normalization_Rules);
+    }
+
+    private function normTerm($sTerm)
+    {
+        if ($this->oNormalizer === null) {
+            return $sTerm;
+        }
+
+        return $this->oNormalizer->transliterate($sTerm);
+    }
+
+    public function setReverseInPlan($bReverse)
+    {
+        $this->bReverseInPlan = $bReverse;
     }
 
     public function setLanguagePreference($aLangPref)
@@ -71,9 +85,7 @@ class Geocode
             $aParams['exclude_place_ids'] = implode(',', $this->aExcludePlaceIDs);
         }
 
-        if ($this->bBoundedSearch) {
-            $aParams['bounded'] = '1';
-        }
+        if ($this->bBoundedSearch) $aParams['bounded'] = '1';
 
         if ($this->aCountryCodes) {
             $aParams['countrycodes'] = implode(',', $this->aCountryCodes);
@@ -88,11 +100,8 @@ class Geocode
 
     public function setLimit($iLimit = 10)
     {
-        if ($iLimit > 50) {
-            $iLimit = 50;
-        } elseif ($iLimit < 1) {
-            $iLimit = 1;
-        }
+        if ($iLimit > 50) $iLimit = 50;
+        if ($iLimit < 1) $iLimit = 1;
 
         $this->iFinalLimit = $iLimit;
         $this->iLimit = $iLimit + min($iLimit, 10);
@@ -187,24 +196,18 @@ class Geocode
         if ($sExcluded) {
             foreach ($sExcluded as $iExcludedPlaceID) {
                 $iExcludedPlaceID = (int)$iExcludedPlaceID;
-                if ($iExcludedPlaceID) {
+                if ($iExcludedPlaceID)
                     $aExcludePlaceIDs[$iExcludedPlaceID] = $iExcludedPlaceID;
-                }
             }
 
-            if (isset($aExcludePlaceIDs)) {
+            if (isset($aExcludePlaceIDs))
                 $this->aExcludePlaceIDs = $aExcludePlaceIDs;
-            }
         }
 
         // Only certain ranks of feature
         $sFeatureType = $oParams->getString('featureType');
-        if (!$sFeatureType) {
-            $sFeatureType = $oParams->getString('featuretype');
-        }
-        if ($sFeatureType) {
-            $this->setFeatureType($sFeatureType);
-        }
+        if (!$sFeatureType) $sFeatureType = $oParams->getString('featuretype');
+        if ($sFeatureType) $this->setFeatureType($sFeatureType);
 
         // Country code list
         $sCountries = $oParams->getStringList('countrycodes');
@@ -214,9 +217,8 @@ class Geocode
                     $aCountries[] = strtolower($sCountryCode);
                 }
             }
-            if (isset($aCountries)) {
+            if (isset($aCountries))
                 $this->aCountryCodes = $aCountries;
-            }
         }
 
         $aViewbox = $oParams->getStringList('viewboxlbrt');
@@ -260,6 +262,7 @@ class Geocode
                 $oParams->getString('country'),
                 $oParams->getString('postalcode')
             );
+            $this->setReverseInPlan(false);
         } else {
             $this->setQuery($sQuery);
         }
@@ -268,17 +271,13 @@ class Geocode
     public function loadStructuredAddressElement($sValue, $sKey, $iNewMinAddressRank, $iNewMaxAddressRank, $aItemListValues)
     {
         $sValue = trim($sValue);
-        if (!$sValue) {
-            return false;
-        }
+        if (!$sValue) return false;
         $this->aStructuredQuery[$sKey] = $sValue;
         if ($this->iMinAddressRank == 0 && $this->iMaxAddressRank == 30) {
             $this->iMinAddressRank = $iNewMinAddressRank;
             $this->iMaxAddressRank = $iNewMaxAddressRank;
         }
-        if ($aItemListValues) {
-            $this->aAddressRankList = array_merge($this->aAddressRankList, $aItemListValues);
-        }
+        if ($aItemListValues) $this->aAddressRankList = array_merge($this->aAddressRankList, $aItemListValues);
         return true;
     }
 
@@ -312,11 +311,11 @@ class Geocode
 
     public function fallbackStructuredQuery()
     {
+        if (!$this->aStructuredQuery) return false;
+
         $aParams = $this->aStructuredQuery;
 
-        if (!$aParams || count($aParams) == 1) {
-            return false;
-        }
+        if (count($aParams) == 1) return false;
 
         $aOrderToFallback = array('postalcode', 'street', 'city', 'county', 'state');
 
@@ -331,7 +330,7 @@ class Geocode
         return false;
     }
 
-    public function getGroupedSearches($aSearches, $aPhrases, $oValidTokens)
+    public function getGroupedSearches($aSearches, $aPhrases, $oValidTokens, $bIsStructured)
     {
         /*
              Calculate all searches using oValidTokens i.e.
@@ -346,26 +345,52 @@ class Geocode
          */
         foreach ($aPhrases as $iPhrase => $oPhrase) {
             $aNewPhraseSearches = array();
-            $oPosition = new SearchPosition(
-                $oPhrase->getPhraseType(),
-                $iPhrase,
-                count($aPhrases)
-            );
+            $sPhraseType = $bIsStructured ? $oPhrase->getPhraseType() : '';
 
             foreach ($oPhrase->getWordSets() as $aWordset) {
                 $aWordsetSearches = $aSearches;
 
                 // Add all words from this wordset
                 foreach ($aWordset as $iToken => $sToken) {
+                    //echo "<br><b>$sToken</b>";
                     $aNewWordsetSearches = array();
-                    $oPosition->setTokenPosition($iToken, count($aWordset));
 
                     foreach ($aWordsetSearches as $oCurrentSearch) {
-                        foreach ($oValidTokens->get($sToken) as $oSearchTerm) {
-                            if ($oSearchTerm->isExtendable($oCurrentSearch, $oPosition)) {
-                                $aNewSearches = $oSearchTerm->extendSearch(
-                                    $oCurrentSearch,
-                                    $oPosition
+                        //echo "<i>";
+                        //var_dump($oCurrentSearch);
+                        //echo "</i>";
+
+                        // Tokens with full name matches.
+                        foreach ($oValidTokens->get(' '.$sToken) as $oSearchTerm) {
+                            $aNewSearches = $oCurrentSearch->extendWithFullTerm(
+                                $oSearchTerm,
+                                $oValidTokens->contains($sToken)
+                                  && strpos($sToken, ' ') === false,
+                                $sPhraseType,
+                                $iToken == 0 && $iPhrase == 0,
+                                $iPhrase == 0,
+                                $iToken + 1 == count($aWordset)
+                                  && $iPhrase + 1 == count($aPhrases)
+                            );
+
+                            foreach ($aNewSearches as $oSearch) {
+                                if ($oSearch->getRank() < $this->iMaxRank) {
+                                    $aNewWordsetSearches[] = $oSearch;
+                                }
+                            }
+                        }
+                        // Look for partial matches.
+                        // Note that there is no point in adding country terms here
+                        // because country is omitted in the address.
+                        if ($sPhraseType != 'country') {
+                            // Allow searching for a word - but at extra cost
+                            foreach ($oValidTokens->get($sToken) as $oSearchTerm) {
+                                $aNewSearches = $oCurrentSearch->extendWithPartialTerm(
+                                    $sToken,
+                                    $oSearchTerm,
+                                    $bIsStructured,
+                                    $iPhrase,
+                                    $oValidTokens->get(' '.$sToken)
                                 );
 
                                 foreach ($aNewSearches as $oSearch) {
@@ -380,6 +405,7 @@ class Geocode
                     usort($aNewWordsetSearches, array('Nominatim\SearchDescription', 'bySearchRank'));
                     $aWordsetSearches = array_slice($aNewWordsetSearches, 0, 50);
                 }
+                //var_Dump('<hr>',count($aWordsetSearches)); exit;
 
                 $aNewPhraseSearches = array_merge($aNewPhraseSearches, $aNewWordsetSearches);
                 usort($aNewPhraseSearches, array('Nominatim\SearchDescription', 'bySearchRank'));
@@ -387,11 +413,8 @@ class Geocode
                 $aSearchHash = array();
                 foreach ($aNewPhraseSearches as $iSearch => $aSearch) {
                     $sHash = serialize($aSearch);
-                    if (isset($aSearchHash[$sHash])) {
-                        unset($aNewPhraseSearches[$iSearch]);
-                    } else {
-                        $aSearchHash[$sHash] = 1;
-                    }
+                    if (isset($aSearchHash[$sHash])) unset($aNewPhraseSearches[$iSearch]);
+                    else $aSearchHash[$sHash] = 1;
                 }
 
                 $aNewPhraseSearches = array_slice($aNewPhraseSearches, 0, 50);
@@ -412,12 +435,10 @@ class Geocode
 
             $iSearchCount = 0;
             $aSearches = array();
-            foreach ($aGroupedSearches as $aNewSearches) {
+            foreach ($aGroupedSearches as $iScore => $aNewSearches) {
                 $iSearchCount += count($aNewSearches);
                 $aSearches = array_merge($aSearches, $aNewSearches);
-                if ($iSearchCount > 50) {
-                    break;
-                }
+                if ($iSearchCount > 50) break;
             }
         }
 
@@ -474,9 +495,7 @@ class Geocode
     public function lookup()
     {
         Debug::newFunction('Geocode::lookup');
-        if (!$this->sQuery && !$this->aStructuredQuery) {
-            return array();
-        }
+        if (!$this->sQuery && !$this->aStructuredQuery) return array();
 
         Debug::printDebugArray('Geocode', $this);
 
@@ -498,10 +517,16 @@ class Geocode
         if ($this->aCountryCodes) {
             $oCtx->setCountryList($this->aCountryCodes);
         }
-        $this->oTokenizer->setCountryRestriction($this->aCountryCodes);
 
         Debug::newSection('Query Preprocessing');
 
+        $sNormQuery = $this->normTerm($this->sQuery);
+        Debug::printVar('Normalized query', $sNormQuery);
+
+        $sLanguagePrefArraySQL = $this->oDB->getArraySQL(
+            $this->oDB->getDBQuotedList($this->aLangPrefOrder)
+        );
+
         $sQuery = $this->sQuery;
         if (!preg_match('//u', $sQuery)) {
             userError('Query string is not UTF-8 encoded.');
@@ -551,62 +576,117 @@ class Geocode
             }
 
             if ($sSpecialTerm && !$aSearches[0]->hasOperator()) {
-                $aTokens = $this->oTokenizer->tokensForSpecialTerm($sSpecialTerm);
+                $sSpecialTerm = pg_escape_string($sSpecialTerm);
+                $sToken = $this->oDB->getOne(
+                    'SELECT make_standard_name(:term)',
+                    array(':term' => $sSpecialTerm),
+                    'Cannot decode query. Wrong encoding?'
+                );
+                $sSQL = 'SELECT class, type FROM word ';
+                $sSQL .= '   WHERE word_token in (\' '.$sToken.'\')';
+                $sSQL .= '   AND class is not null AND class not in (\'place\')';
 
-                if (!empty($aTokens)) {
-                    $aNewSearches = array();
-                    $oPosition = new SearchPosition('', 0, 1);
-                    $oPosition->setTokenPosition(0, 1);
-
-                    foreach ($aSearches as $oSearch) {
-                        foreach ($aTokens as $oToken) {
-                            $aNewSearches = array_merge(
-                                $aNewSearches,
-                                $oToken->extendSearch($oSearch, $oPosition)
-                            );
-                        }
+                Debug::printSQL($sSQL);
+                $aSearchWords = $this->oDB->getAll($sSQL);
+                $aNewSearches = array();
+                foreach ($aSearches as $oSearch) {
+                    foreach ($aSearchWords as $aSearchTerm) {
+                        $oNewSearch = clone $oSearch;
+                        $oNewSearch->setPoiSearch(
+                            Operator::TYPE,
+                            $aSearchTerm['class'],
+                            $aSearchTerm['type']
+                        );
+                        $aNewSearches[] = $oNewSearch;
                     }
-                    $aSearches = $aNewSearches;
                 }
+                $aSearches = $aNewSearches;
             }
 
             // Split query into phrases
             // Commas are used to reduce the search space by indicating where phrases split
-            $aPhrases = array();
             if ($this->aStructuredQuery) {
-                foreach ($this->aStructuredQuery as $iPhrase => $sPhrase) {
-                    $aPhrases[] = new Phrase($sPhrase, $iPhrase);
-                }
+                $aInPhrases = $this->aStructuredQuery;
+                $bStructuredPhrases = true;
             } else {
-                foreach (explode(',', $sQuery) as $sPhrase) {
-                    $aPhrases[] = new Phrase($sPhrase, '');
-                }
+                $aInPhrases = explode(',', $sQuery);
+                $bStructuredPhrases = false;
             }
 
             Debug::printDebugArray('Search context', $oCtx);
             Debug::printDebugArray('Base search', empty($aSearches) ? null : $aSearches[0]);
+            Debug::printVar('Final query phrases', $aInPhrases);
 
+            // Convert each phrase to standard form
+            // Create a list of standard words
+            // Get all 'sets' of words
+            // Generate a complete list of all
             Debug::newSection('Tokenization');
-            $oValidTokens = $this->oTokenizer->extractTokensFromPhrases($aPhrases);
+            $aTokens = array();
+            $aPhrases = array();
+            foreach ($aInPhrases as $iPhrase => $sPhrase) {
+                $sPhrase = $this->oDB->getOne(
+                    'SELECT make_standard_name(:phrase)',
+                    array(':phrase' => $sPhrase),
+                    'Cannot normalize query string (is it a UTF-8 string?)'
+                );
+                if (trim($sPhrase)) {
+                    $oPhrase = new Phrase($sPhrase, is_string($iPhrase) ? $iPhrase : '');
+                    $oPhrase->addTokens($aTokens);
+                    $aPhrases[] = $oPhrase;
+                }
+            }
+
+            Debug::printVar('Tokens', $aTokens);
+
+            $oValidTokens = new TokenList();
+
+            if (!empty($aTokens)) {
+                $oValidTokens->addTokensFromDB(
+                    $this->oDB,
+                    $aTokens,
+                    $this->aCountryCodes,
+                    $sNormQuery,
+                    $this->oNormalizer
+                );
 
-            if ($oValidTokens->count() > 0) {
                 $oCtx->setFullNameWords($oValidTokens->getFullWordIDs());
 
-                $aPhrases = array_filter($aPhrases, function ($oPhrase) {
-                    return $oPhrase->getWordSets() !== null;
-                });
+                // Try more interpretations for Tokens that could not be matched.
+                foreach ($aTokens as $sToken) {
+                    if ($sToken[0] == ' ' && !$oValidTokens->contains($sToken)) {
+                        if (preg_match('/^ ([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) {
+                            // US ZIP+4 codes - merge in the 5-digit ZIP code
+                            $oValidTokens->addToken(
+                                $sToken,
+                                new Token\Postcode(null, $aData[1], 'us')
+                            );
+                        } elseif (preg_match('/^ [0-9]+$/', $sToken)) {
+                            // Unknown single word token with a number.
+                            // Assume it is a house number.
+                            $oValidTokens->addToken(
+                                $sToken,
+                                new Token\HouseNumber(null, trim($sToken))
+                            );
+                        }
+                    }
+                }
 
                 // Any words that have failed completely?
                 // TODO: suggestions
 
                 Debug::printGroupTable('Valid Tokens', $oValidTokens->debugInfo());
+
+                foreach ($aPhrases as $oPhrase) {
+                    $oPhrase->computeWordSets($oValidTokens);
+                }
                 Debug::printDebugTable('Phrases', $aPhrases);
 
                 Debug::newSection('Search candidates');
 
-                $aGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $oValidTokens);
+                $aGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $oValidTokens, $bStructuredPhrases);
 
-                if (!$this->aStructuredQuery) {
+                if ($this->bReverseInPlan) {
                     // Reverse phrase array and also reverse the order of the wordsets in
                     // the first and final phrase. Don't bother about phrases in the middle
                     // because order in the address doesn't matter.
@@ -615,7 +695,7 @@ class Geocode
                     if (count($aPhrases) > 1) {
                         $aPhrases[count($aPhrases)-1]->invertWordSets();
                     }
-                    $aReverseGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $oValidTokens);
+                    $aReverseGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $oValidTokens, false);
 
                     foreach ($aGroupedSearches as $aSearches) {
                         foreach ($aSearches as $aSearch) {
@@ -634,9 +714,7 @@ class Geocode
                 $aGroupedSearches = array();
                 foreach ($aSearches as $aSearch) {
                     if ($aSearch->getRank() < $this->iMaxRank) {
-                        if (!isset($aGroupedSearches[$aSearch->getRank()])) {
-                            $aGroupedSearches[$aSearch->getRank()] = array();
-                        }
+                        if (!isset($aGroupedSearches[$aSearch->getRank()])) $aGroupedSearches[$aSearch->getRank()] = array();
                         $aGroupedSearches[$aSearch->getRank()][] = $aSearch;
                     }
                 }
@@ -650,9 +728,7 @@ class Geocode
                     $sHash = serialize($aSearch);
                     if (isset($aSearchHash[$sHash])) {
                         unset($aGroupedSearches[$iGroup][$iSearch]);
-                        if (empty($aGroupedSearches[$iGroup])) {
-                            unset($aGroupedSearches[$iGroup]);
-                        }
+                        if (empty($aGroupedSearches[$iGroup])) unset($aGroupedSearches[$iGroup]);
                     } else {
                         $aSearchHash[$sHash] = 1;
                     }
@@ -696,9 +772,7 @@ class Geocode
                         }
                     }
 
-                    if ($iQueryLoop > 20) {
-                        break;
-                    }
+                    if ($iQueryLoop > 20) break;
                 }
 
                 if (!empty($aResults)) {
@@ -764,6 +838,7 @@ class Geocode
                     foreach ($aResults as $oResult) {
                         if (($this->iMaxAddressRank == 30 &&
                              ($oResult->iTable == Result::TABLE_OSMLINE
+                              || $oResult->iTable == Result::TABLE_AUX
                               || $oResult->iTable == Result::TABLE_TIGER))
                             || in_array($oResult->iId, $aFilteredIDs)
                         ) {
@@ -773,9 +848,9 @@ class Geocode
                     $aResults = $tempIDs;
                 }
 
-                if (!empty($aResults) || $iGroupLoop > 4 || $iQueryLoop > 30) {
-                    break;
-                }
+                if (!empty($aResults)) break;
+                if ($iGroupLoop > 4) break;
+                if ($iQueryLoop > 30) break;
             }
         } else {
             // Just interpret as a reverse geocode
@@ -793,8 +868,10 @@ class Geocode
 
         // No results? Done
         if (empty($aResults)) {
-            if ($this->bFallback && $this->fallbackStructuredQuery()) {
-                return $this->lookup();
+            if ($this->bFallback) {
+                if ($this->fallbackStructuredQuery()) {
+                    return $this->lookup();
+                }
             }
 
             return array();
@@ -813,9 +890,7 @@ class Geocode
 
         $aRecheckWords = preg_split('/\b[\s,\\-]*/u', $sQuery);
         foreach ($aRecheckWords as $i => $sWord) {
-            if (!preg_match('/[\pL\pN]/', $sWord)) {
-                unset($aRecheckWords[$i]);
-            }
+            if (!preg_match('/[\pL\pN]/', $sWord)) unset($aRecheckWords[$i]);
         }
 
         Debug::printVar('Recheck words', $aRecheckWords);
@@ -875,9 +950,7 @@ class Geocode
                 foreach ($aRecheckWords as $i => $sWord) {
                     if (stripos($sAddress, $sWord)!==false) {
                         $iCountWords++;
-                        if (preg_match('/(^|,)\s*'.preg_quote($sWord, '/').'\s*(,|$)/', $sAddress)) {
-                            $iCountWords += 0.1;
-                        }
+                        if (preg_match('/(^|,)\s*'.preg_quote($sWord, '/').'\s*(,|$)/', $sAddress)) $iCountWords += 0.1;
                     }
                 }
 
@@ -894,8 +967,15 @@ class Geocode
         $aToFilter = $aSearchResults;
         $aSearchResults = array();
 
+        $bFirst = true;
         foreach ($aToFilter as $aResult) {
             $this->aExcludePlaceIDs[$aResult['place_id']] = $aResult['place_id'];
+            if ($bFirst) {
+                $fLat = $aResult['lat'];
+                $fLon = $aResult['lon'];
+                if (isset($aResult['zoom'])) $iZoom = $aResult['zoom'];
+                $bFirst = false;
+            }
             if (!$this->oPlaceLookup->doDeDupe() || (!isset($aOSMIDDone[$aResult['osm_type'].$aResult['osm_id']])
                 && !isset($aClassTypeNameDone[$aResult['osm_type'].$aResult['class'].$aResult['type'].$aResult['name'].$aResult['admin_level']]))
             ) {
@@ -905,9 +985,7 @@ class Geocode
             }
 
             // Absolute limit on number of results
-            if (count($aSearchResults) >= $this->iFinalLimit) {
-                break;
-            }
+            if (count($aSearchResults) >= $this->iFinalLimit) break;
         }
 
         Debug::printVar('Post-filter results', $aSearchResults);
@@ -921,6 +999,7 @@ class Geocode
                 'Structured query' => $this->aStructuredQuery,
                 'Name keys' => Debug::fmtArrayVals($this->aLangPrefOrder),
                 'Excluded place IDs' => Debug::fmtArrayVals($this->aExcludePlaceIDs),
+                'Try reversed query'=> $this->bReverseInPlan,
                 'Limit (for searches)' => $this->iLimit,
                 'Limit (for results)'=> $this->iFinalLimit,
                 'Country codes' => Debug::fmtArrayVals($this->aCountryCodes),

lib-php/ParameterParser.php

@@ -90,16 +90,14 @@ class ParameterParser
         $aLanguages = array();
         $sLangString = $this->getString('accept-language', $sFallback);
 
-        if ($sLangString
-            && preg_match_all('/(([a-z]{1,8})([-_][a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i', $sLangString, $aLanguagesParse, PREG_SET_ORDER)
-        ) {
-            foreach ($aLanguagesParse as $iLang => $aLanguage) {
-                $aLanguages[$aLanguage[1]] = isset($aLanguage[5])?(float)$aLanguage[5]:1 - ($iLang/100);
-                if (!isset($aLanguages[$aLanguage[2]])) {
-                    $aLanguages[$aLanguage[2]] = $aLanguages[$aLanguage[1]]/10;
+        if ($sLangString) {
+            if (preg_match_all('/(([a-z]{1,8})([-_][a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i', $sLangString, $aLanguagesParse, PREG_SET_ORDER)) {
+                foreach ($aLanguagesParse as $iLang => $aLanguage) {
+                    $aLanguages[$aLanguage[1]] = isset($aLanguage[5])?(float)$aLanguage[5]:1 - ($iLang/100);
+                    if (!isset($aLanguages[$aLanguage[2]])) $aLanguages[$aLanguage[2]] = $aLanguages[$aLanguage[1]]/10;
                 }
+                arsort($aLanguages);
             }
-            arsort($aLanguages);
         }
         if (empty($aLanguages) && CONST_Default_Language) {
             $aLanguages[CONST_Default_Language] = 1;

lib-php/Phrase.php

@@ -16,6 +16,8 @@ class Phrase
     private $sPhrase;
     // Element type for structured searches.
     private $sPhraseType;
+    // Space-separated words of the phrase.
+    private $aWords;
     // Possible segmentations of the phrase.
     private $aWordSets;
 
@@ -36,14 +38,7 @@ class Phrase
     {
         $this->sPhrase = trim($sPhrase);
         $this->sPhraseType = $sPhraseType;
-    }
-
-    /**
-     * Get the orginal phrase of the string.
-     */
-    public function getPhrase()
-    {
-        return $this->sPhrase;
+        $this->aWords = explode(' ', $this->sPhrase);
     }
 
     /**
@@ -68,6 +63,30 @@ class Phrase
         return $this->aWordSets;
     }
 
+    /**
+     * Add the tokens from this phrase to the given list of tokens.
+     *
+     * @param string[] $aTokens List of tokens to append.
+     *
+     * @return void
+     */
+    public function addTokens(&$aTokens)
+    {
+        $iNumWords = count($this->aWords);
+
+        for ($i = 0; $i < $iNumWords; $i++) {
+            $sPhrase = $this->aWords[$i];
+            $aTokens[' '.$sPhrase] = ' '.$sPhrase;
+            $aTokens[$sPhrase] = $sPhrase;
+
+            for ($j = $i + 1; $j < $iNumWords; $j++) {
+                $sPhrase .= ' '.$this->aWords[$j];
+                $aTokens[' '.$sPhrase] = ' '.$sPhrase;
+                $aTokens[$sPhrase] = $sPhrase;
+            }
+        }
+    }
+
     /**
      * Invert the set of possible segmentations.
      *
@@ -80,27 +99,21 @@ class Phrase
         }
     }
 
-    public function computeWordSets($aWords, $oTokens)
+    public function computeWordSets($oTokens)
     {
-        $iNumWords = count($aWords);
-
-        if ($iNumWords == 0) {
-            $this->aWordSets = null;
-            return;
-        }
-
+        $iNumWords = count($this->aWords);
         // Caches the word set for the partial phrase up to word i.
         $aSetCache = array_fill(0, $iNumWords, array());
 
         // Initialise first element of cache. There can only be the word.
-        if ($oTokens->containsAny($aWords[0])) {
-            $aSetCache[0][] = array($aWords[0]);
+        if ($oTokens->containsAny($this->aWords[0])) {
+            $aSetCache[0][] = array($this->aWords[0]);
         }
 
         // Now do the next elements using what we already have.
         for ($i = 1; $i < $iNumWords; $i++) {
             for ($j = $i; $j > 0; $j--) {
-                $sPartial = $j == $i ? $aWords[$j] : $aWords[$j].' '.$sPartial;
+                $sPartial = $j == $i ? $this->aWords[$j] : $this->aWords[$j].' '.$sPartial;
                 if (!empty($aSetCache[$j - 1]) && $oTokens->containsAny($sPartial)) {
                     $aPartial = array($sPartial);
                     foreach ($aSetCache[$j - 1] as $aSet) {
@@ -123,7 +136,7 @@ class Phrase
             }
 
             // finally the current full phrase
-            $sPartial = $aWords[0].' '.$sPartial;
+            $sPartial = $this->aWords[0].' '.$sPartial;
             if ($oTokens->containsAny($sPartial)) {
                 $aSetCache[$i][] = array($sPartial);
             }
@@ -140,6 +153,7 @@ class Phrase
         return array(
                 'Type' => $this->sPhraseType,
                 'Phrase' => $this->sPhrase,
+                'Words' => $this->aWords,
                 'WordSets' => $this->aWordSets
                );
     }

lib-php/PlaceLookup.php

@@ -89,36 +89,20 @@ class PlaceLookup
     {
         $aParams = array();
 
-        if ($this->bAddressDetails) {
-            $aParams['addressdetails'] = '1';
-        }
-        if ($this->bExtraTags) {
-            $aParams['extratags'] = '1';
-        }
-        if ($this->bNameDetails) {
-            $aParams['namedetails'] = '1';
-        }
+        if ($this->bAddressDetails) $aParams['addressdetails'] = '1';
+        if ($this->bExtraTags) $aParams['extratags'] = '1';
+        if ($this->bNameDetails) $aParams['namedetails'] = '1';
 
-        if ($this->bIncludePolygonAsText) {
-            $aParams['polygon_text'] = '1';
-        }
-        if ($this->bIncludePolygonAsGeoJSON) {
-            $aParams['polygon_geojson'] = '1';
-        }
-        if ($this->bIncludePolygonAsKML) {
-            $aParams['polygon_kml'] = '1';
-        }
-        if ($this->bIncludePolygonAsSVG) {
-            $aParams['polygon_svg'] = '1';
-        }
+        if ($this->bIncludePolygonAsText) $aParams['polygon_text'] = '1';
+        if ($this->bIncludePolygonAsGeoJSON) $aParams['polygon_geojson'] = '1';
+        if ($this->bIncludePolygonAsKML) $aParams['polygon_kml'] = '1';
+        if ($this->bIncludePolygonAsSVG) $aParams['polygon_svg'] = '1';
 
         if ($this->fPolygonSimplificationThreshold > 0.0) {
             $aParams['polygon_threshold'] = $this->fPolygonSimplificationThreshold;
         }
 
-        if (!$this->bDeDupe) {
-            $aParams['dedupe'] = '0';
-        }
+        if (!$this->bDeDupe) $aParams['dedupe'] = '0';
 
         return $aParams;
     }
@@ -163,9 +147,8 @@ class PlaceLookup
 
     private function langAddressSql($sHousenumber)
     {
-        if ($this->bAddressDetails) {
+        if ($this->bAddressDetails)
             return ''; // langaddress will be computed from address details
-        }
 
         return 'get_address_by_language(place_id,'.$sHousenumber.','.$this->aLangPrefOrderSql.') AS langaddress,';
     }
@@ -251,20 +234,12 @@ class PlaceLookup
             $sSQL .= '     housenumber,';
             $sSQL .= '     country_code, ';
             $sSQL .= '     importance, ';
-            if (!$this->bDeDupe) {
-                $sSQL .= 'place_id,';
-            }
-            if (!$this->bAddressDetails) {
-                $sSQL .= 'langaddress, ';
-            }
+            if (!$this->bDeDupe) $sSQL .= 'place_id,';
+            if (!$this->bAddressDetails) $sSQL .= 'langaddress, ';
             $sSQL .= '     placename, ';
             $sSQL .= '     ref, ';
-            if ($this->bExtraTags) {
-                $sSQL .= 'extratags, ';
-            }
-            if ($this->bNameDetails) {
-                $sSQL .= 'name, ';
-            }
+            if ($this->bExtraTags) $sSQL .= 'extratags, ';
+            if ($this->bNameDetails) $sSQL .= 'name, ';
             $sSQL .= '     extra_place ';
 
             $aSubSelects[] = $sSQL;
@@ -285,12 +260,8 @@ class PlaceLookup
             $sSQL .= $this->langAddressSql('-1');
             $sSQL .= '  postcode as placename,';
             $sSQL .= '  postcode as ref,';
-            if ($this->bExtraTags) {
-                $sSQL .= 'null::text AS extra,';
-            }
-            if ($this->bNameDetails) {
-                $sSQL .= 'null::text AS names,';
-            }
+            if ($this->bExtraTags) $sSQL .= 'null::text AS extra,';
+            if ($this->bNameDetails) $sSQL .= 'null::text AS names,';
             $sSQL .= '  ST_x(geometry) AS lon, ST_y(geometry) AS lat,';
             $sSQL .= '  (0.75-(rank_search::float/40)) AS importance, ';
             $sSQL .= $this->addressImportanceSql('geometry', 'lp.parent_place_id');
@@ -327,12 +298,8 @@ class PlaceLookup
                     $sSQL .= $this->langAddressSql('housenumber_for_place');
                     $sSQL .= '     null::text AS placename, ';
                     $sSQL .= '     null::text AS ref, ';
-                    if ($this->bExtraTags) {
-                        $sSQL .= 'null::text AS extra,';
-                    }
-                    if ($this->bNameDetails) {
-                        $sSQL .= 'null::text AS names,';
-                    }
+                    if ($this->bExtraTags) $sSQL .= 'null::text AS extra,';
+                    if ($this->bNameDetails) $sSQL .= 'null::text AS names,';
                     $sSQL .= '     st_x(centroid) AS lon, ';
                     $sSQL .= '     st_y(centroid) AS lat,';
                     $sSQL .= '     -1.15 AS importance, ';
@@ -377,12 +344,8 @@ class PlaceLookup
                 $sSQL .= $this->langAddressSql('housenumber_for_place');
                 $sSQL .= '  null::text AS placename, ';
                 $sSQL .= '  null::text AS ref, ';
-                if ($this->bExtraTags) {
-                    $sSQL .= 'null::text AS extra, ';
-                }
-                if ($this->bNameDetails) {
-                    $sSQL .= 'null::text AS names, ';
-                }
+                if ($this->bExtraTags) $sSQL .= 'null::text AS extra, ';
+                if ($this->bNameDetails) $sSQL .= 'null::text AS names, ';
                 $sSQL .= '  st_x(centroid) AS lon, ';
                 $sSQL .= '  st_y(centroid) AS lat, ';
                 // slightly smaller than the importance for normal houses
@@ -410,6 +373,42 @@ class PlaceLookup
 
                 $aSubSelects[] = $sSQL;
             }
+
+            if (CONST_Use_Aux_Location_data) {
+                $sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_AUX);
+                if ($sPlaceIDs) {
+                    $sHousenumbers = Result::sqlHouseNumberTable($aResults, Result::TABLE_AUX);
+                    $sSQL = '  SELECT ';
+                    $sSQL .= "     'L' AS osm_type, ";
+                    $sSQL .= '     place_id AS osm_id, ';
+                    $sSQL .= "     'place' AS class,";
+                    $sSQL .= "     'house' AS type, ";
+                    $sSQL .= '     null::smallint AS admin_level, ';
+                    $sSQL .= '     30 AS rank_search,';
+                    $sSQL .= '     30 AS rank_address, ';
+                    $sSQL .= '     place_id,';
+                    $sSQL .= '     parent_place_id, ';
+                    $sSQL .= '     housenumber,';
+                    $sSQL .= "     'us' AS country_code, ";
+                    $sSQL .= $this->langAddressSql('-1');
+                    $sSQL .= '     null::text AS placename, ';
+                    $sSQL .= '     null::text AS ref, ';
+                    if ($this->bExtraTags) $sSQL .= 'null::text AS extra, ';
+                    if ($this->bNameDetails) $sSQL .= 'null::text AS names, ';
+                    $sSQL .= '     ST_X(centroid) AS lon, ';
+                    $sSQL .= '     ST_Y(centroid) AS lat, ';
+                    $sSQL .= '     -1.10 AS importance, ';
+                    $sSQL .= $this->addressImportanceSql(
+                        'centroid',
+                        'location_property_aux.parent_place_id'
+                    );
+                    $sSQL .= '     null::text AS extra_place ';
+                    $sSQL .= '  FROM location_property_aux ';
+                    $sSQL .= "  WHERE place_id in ($sPlaceIDs) ";
+
+                    $aSubSelects[] = $sSQL;
+                }
+            }
         }
 
         if (empty($aSubSelects)) {
@@ -485,9 +484,7 @@ class PlaceLookup
     {
 
         $aOutlineResult = array();
-        if (!$iPlaceID) {
-            return $aOutlineResult;
-        }
+        if (!$iPlaceID) return $aOutlineResult;
 
         // Get the bounding box and outline polygon
         $sSQL = 'select place_id,0 as numfeatures,st_area(geometry) as area,';
@@ -499,18 +496,10 @@ class PlaceLookup
         }
         $sSQL .= ' ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,';
         $sSQL .= ' ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon';
-        if ($this->bIncludePolygonAsGeoJSON) {
-            $sSQL .= ',ST_AsGeoJSON(geometry) as asgeojson';
-        }
-        if ($this->bIncludePolygonAsKML) {
-            $sSQL .= ',ST_AsKML(geometry) as askml';
-        }
-        if ($this->bIncludePolygonAsSVG) {
-            $sSQL .= ',ST_AsSVG(geometry) as assvg';
-        }
-        if ($this->bIncludePolygonAsText) {
-            $sSQL .= ',ST_AsText(geometry) as astext';
-        }
+        if ($this->bIncludePolygonAsGeoJSON) $sSQL .= ',ST_AsGeoJSON(geometry) as asgeojson';
+        if ($this->bIncludePolygonAsKML) $sSQL .= ',ST_AsKML(geometry) as askml';
+        if ($this->bIncludePolygonAsSVG) $sSQL .= ',ST_AsSVG(geometry) as assvg';
+        if ($this->bIncludePolygonAsText) $sSQL .= ',ST_AsText(geometry) as astext';
         if ($fLonReverse != null && $fLatReverse != null) {
             $sFrom = ' from (SELECT * , CASE WHEN (class = \'highway\') AND (ST_GeometryType(geometry) = \'ST_LineString\') THEN ';
             $sFrom .=' ST_ClosestPoint(geometry, ST_SetSRID(ST_Point('.$fLatReverse.','.$fLonReverse.'),4326))';
@@ -533,18 +522,10 @@ class PlaceLookup
                 $aOutlineResult['lon'] = $aPointPolygon['centrelon'];
             }
 
-            if ($this->bIncludePolygonAsGeoJSON) {
-                $aOutlineResult['asgeojson'] = $aPointPolygon['asgeojson'];
-            }
-            if ($this->bIncludePolygonAsKML) {
-                $aOutlineResult['askml'] = $aPointPolygon['askml'];
-            }
-            if ($this->bIncludePolygonAsSVG) {
-                $aOutlineResult['assvg'] = $aPointPolygon['assvg'];
-            }
-            if ($this->bIncludePolygonAsText) {
-                $aOutlineResult['astext'] = $aPointPolygon['astext'];
-            }
+            if ($this->bIncludePolygonAsGeoJSON) $aOutlineResult['asgeojson'] = $aPointPolygon['asgeojson'];
+            if ($this->bIncludePolygonAsKML) $aOutlineResult['askml'] = $aPointPolygon['askml'];
+            if ($this->bIncludePolygonAsSVG) $aOutlineResult['assvg'] = $aPointPolygon['assvg'];
+            if ($this->bIncludePolygonAsText) $aOutlineResult['astext'] = $aPointPolygon['astext'];
 
             if (abs($aPointPolygon['minlat'] - $aPointPolygon['maxlat']) < 0.0000001) {
                 $aPointPolygon['minlat'] = $aPointPolygon['minlat'] - $fRadius;

lib-php/Result.php

@@ -13,7 +13,8 @@ class Result
     const TABLE_PLACEX = 0;
     const TABLE_POSTCODE = 1;
     const TABLE_OSMLINE = 2;
-    const TABLE_TIGER = 3;
+    const TABLE_AUX = 3;
+    const TABLE_TIGER = 4;
 
     /// Database table that contains the result.
     public $iTable;
@@ -55,27 +56,6 @@ class Result
             }
         )));
     }
-
-    public static function joinIdsByTableMinRank($aResults, $iTable, $iMinAddressRank)
-    {
-        return join(',', array_keys(array_filter(
-            $aResults,
-            function ($aValue) use ($iTable, $iMinAddressRank) {
-                return $aValue->iTable == $iTable && $aValue->iAddressRank >= $iMinAddressRank;
-            }
-        )));
-    }
-
-    public static function joinIdsByTableMaxRank($aResults, $iTable, $iMaxAddressRank)
-    {
-        return join(',', array_keys(array_filter(
-            $aResults,
-            function ($aValue) use ($iTable, $iMaxAddressRank) {
-                return $aValue->iTable == $iTable && $aValue->iAddressRank <= $iMaxAddressRank;
-            }
-        )));
-    }
-
     public static function sqlHouseNumberTable($aResults, $iTable)
     {
         $sHousenumbers = '';

lib-php/ReverseGeocode.php

@@ -74,6 +74,8 @@ class ReverseGeocode
 
     protected function lookupLargeArea($sPointSQL, $iMaxRank)
     {
+        $oResult = null;
+
         if ($iMaxRank > 4) {
             $aPlace = $this->lookupPolygon($sPointSQL, $iMaxRank);
             if ($aPlace) {
@@ -165,13 +167,9 @@ class ReverseGeocode
     {
         Debug::newFunction('lookupPolygon');
         // polygon search begins at suburb-level
-        if ($iMaxRank > 25) {
-            $iMaxRank = 25;
-        }
+        if ($iMaxRank > 25) $iMaxRank = 25;
         // no polygon search over country-level
-        if ($iMaxRank < 5) {
-            $iMaxRank = 5;
-        }
+        if ($iMaxRank < 5) $iMaxRank = 5;
         // search for polygon
         $sSQL = 'SELECT place_id, parent_place_id, rank_address, rank_search FROM';
         $sSQL .= '(select place_id, parent_place_id, rank_address, rank_search, country_code, geometry';
@@ -192,6 +190,7 @@ class ReverseGeocode
 
         if ($aPoly) {
         // if a polygon is found, search for placenodes begins ...
+            $iParentPlaceID = $aPoly['parent_place_id'];
             $iRankAddress = $aPoly['rank_address'];
             $iRankSearch = $aPoly['rank_search'];
             $iPlaceID = $aPoly['place_id'];
@@ -243,24 +242,26 @@ class ReverseGeocode
     public function lookupPoint($sPointSQL, $bDoInterpolation = true)
     {
         Debug::newFunction('lookupPoint');
+        // starts if the search is on POI or street level,
+        // searches for the nearest POI or street,
+        // if a street is found and a POI is searched for,
+        // the nearest POI which the found street is a parent of is choosen.
+        $iMaxRank = $this->iMaxRank;
+
         // Find the nearest point
         $fSearchDiam = 0.006;
         $oResult = null;
         $aPlace = null;
 
         // for POI or street level
-        if ($this->iMaxRank >= 26) {
-            // starts if the search is on POI or street level,
-            // searches for the nearest POI or street,
-            // if a street is found and a POI is searched for,
-            // the nearest POI which the found street is a parent of is choosen.
+        if ($iMaxRank >= 26) {
             $sSQL = 'select place_id,parent_place_id,rank_address,country_code,';
             $sSQL .= ' ST_distance('.$sPointSQL.', geometry) as distance';
             $sSQL .= ' FROM ';
             $sSQL .= ' placex';
             $sSQL .= '   WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')';
             $sSQL .= '   AND';
-            $sSQL .= ' rank_address between 26 and '.$this->iMaxRank;
+            $sSQL .= ' rank_address between 26 and '.$iMaxRank;
             $sSQL .= ' and (name is not null or housenumber is not null';
             $sSQL .= ' or rank_address between 26 and 27)';
             $sSQL .= ' and (rank_address between 26 and 27';
@@ -283,7 +284,7 @@ class ReverseGeocode
 
             if ($aPlace) {
                 // if street and maxrank > streetlevel
-                if ($iRankAddress <= 27 && $this->iMaxRank > 27) {
+                if ($iRankAddress <= 27 && $iMaxRank > 27) {
                     // find the closest object (up to a certain radius) of which the street is a parent of
                     $sSQL = ' select place_id,';
                     $sSQL .= ' ST_distance('.$sPointSQL.', geometry) as distance';
@@ -337,7 +338,7 @@ class ReverseGeocode
                 }
             }
 
-            if ($bDoInterpolation && $this->iMaxRank >= 30) {
+            if ($bDoInterpolation && $iMaxRank >= 30) {
                 $fDistance = $fSearchDiam;
                 if ($aPlace) {
                     // We can't reliably go from the closest street to an
@@ -355,6 +356,7 @@ class ReverseGeocode
                     $oResult = new Result($aHouse['place_id'], Result::TABLE_OSMLINE);
                     $oResult->iHouseNumber = closestHouseNumber($aHouse);
                     $aPlace = $aHouse;
+                    $iRankAddress = 30;
                 }
             }
 
@@ -364,7 +366,7 @@ class ReverseGeocode
             }
         } else {
             // lower than street level ($iMaxRank < 26 )
-            $oResult = $this->lookupLargeArea($sPointSQL, $this->iMaxRank);
+            $oResult = $this->lookupLargeArea($sPointSQL, $iMaxRank);
         }
 
         Debug::printVar('Final result', $oResult);

lib-php/SearchDescription.php

@@ -67,6 +67,35 @@ class SearchDescription
         return $this->iSearchRank;
     }
 
+    /**
+     * Make this search a POI search.
+     *
+     * In a POI search, objects are not (only) searched by their name
+     * but also by the primary OSM key/value pair (class and type in Nominatim).
+     *
+     * @param integer $iOperator Type of POI search
+     * @param string  $sClass    Class (or OSM tag key) of POI.
+     * @param string  $sType     Type (or OSM tag value) of POI.
+     *
+     * @return void
+     */
+    public function setPoiSearch($iOperator, $sClass, $sType)
+    {
+        $this->iOperator = $iOperator;
+        $this->sClass = $sClass;
+        $this->sType = $sType;
+    }
+
+    /**
+     * Check if any operator is set.
+     *
+     * @return bool True, if this is a special search operation.
+     */
+    public function hasOperator()
+    {
+        return $this->iOperator != Operator::NONE;
+    }
+
     /**
      * Extract key/value pairs from a query.
      *
@@ -119,234 +148,251 @@ class SearchDescription
 
     /////////// Search building functions
 
-    /**
-     * Create a copy of this search description adding to search rank.
-     *
-     * @param integer $iTermCost  Cost to add to the current search rank.
-     *
-     * @return object Cloned search description.
-     */
-    public function clone($iTermCost)
-    {
-        $oSearch = clone $this;
-        $oSearch->iSearchRank += $iTermCost;
-
-        return $oSearch;
-    }
 
     /**
-     * Check if the search currently includes a name.
+     * Derive new searches by adding a full term to the existing search.
      *
-     * @param bool bIncludeNonNames  If true stop-word tokens are taken into
-     *                               account, too.
+     * @param object $oSearchTerm  Description of the token.
+     * @param bool   $bHasPartial  True if there are also tokens of partial terms
+     *                             with the same name.
+     * @param string $sPhraseType  Type of phrase the token is contained in.
+     * @param bool   $bFirstToken  True if the token is at the beginning of the
+     *                             query.
+     * @param bool   $bFirstPhrase True if the token is in the first phrase of
+     *                             the query.
+     * @param bool   $bLastToken   True if the token is at the end of the query.
      *
-     * @return bool True, if search has a name.
+     * @return SearchDescription[] List of derived search descriptions.
      */
-    public function hasName($bIncludeNonNames = false)
+    public function extendWithFullTerm($oSearchTerm, $bHasPartial, $sPhraseType, $bFirstToken, $bFirstPhrase, $bLastToken)
     {
-        return !empty($this->aName)
-               || (!empty($this->aNameNonSearch) && $bIncludeNonNames);
-    }
+        $aNewSearches = array();
 
-    /**
-     * Check if the search currently includes an address term.
-     *
-     * @return bool True, if any address term is included, including stop-word
-     *              terms.
-     */
-    public function hasAddress()
-    {
-        return !empty($this->aAddress) || !empty($this->aAddressNonSearch);
-    }
+        if (($sPhraseType == '' || $sPhraseType == 'country')
+            && is_a($oSearchTerm, '\Nominatim\Token\Country')
+        ) {
+            if (!$this->sCountryCode) {
+                $oSearch = clone $this;
+                $oSearch->iSearchRank++;
+                $oSearch->sCountryCode = $oSearchTerm->sCountryCode;
+                // Country is almost always at the end of the string
+                // - increase score for finding it anywhere else (optimisation)
+                if (!$bLastToken) {
+                    $oSearch->iSearchRank += 5;
+                    $oSearch->iNamePhrase = -1;
+                }
+                $aNewSearches[] = $oSearch;
+            }
+        } elseif (($sPhraseType == '' || $sPhraseType == 'postalcode')
+                  && is_a($oSearchTerm, '\Nominatim\Token\Postcode')
+        ) {
+            if (!$this->sPostcode) {
+                // If we have structured search or this is the first term,
+                // make the postcode the primary search element.
+                if ($this->iOperator == Operator::NONE && $bFirstToken) {
+                    $oSearch = clone $this;
+                    $oSearch->iSearchRank++;
+                    $oSearch->iOperator = Operator::POSTCODE;
+                    $oSearch->aAddress = array_merge($this->aAddress, $this->aName);
+                    $oSearch->aName =
+                        array($oSearchTerm->iId => $oSearchTerm->sPostcode);
+                    $aNewSearches[] = $oSearch;
+                }
 
-    /**
-     * Check if a country restriction is currently included in the search.
-     *
-     * @return bool True, if a country restriction is set.
-     */
-    public function hasCountry()
-    {
-        return $this->sCountryCode !== '';
-    }
+                // If we have a structured search or this is not the first term,
+                // add the postcode as an addendum.
+                if ($this->iOperator != Operator::POSTCODE
+                    && ($sPhraseType == 'postalcode' || !empty($this->aName))
+                ) {
+                    $oSearch = clone $this;
+                    $oSearch->iSearchRank++;
+                    $oSearch->iNamePhrase = -1;
+                    if (strlen($oSearchTerm->sPostcode) < 4) {
+                        $oSearch->iSearchRank += 4 - strlen($oSearchTerm->sPostcode);
+                    }
+                    $oSearch->sPostcode = $oSearchTerm->sPostcode;
+                    $aNewSearches[] = $oSearch;
+                }
+            }
+        } elseif (($sPhraseType == '' || $sPhraseType == 'street')
+                 && is_a($oSearchTerm, '\Nominatim\Token\HouseNumber')
+        ) {
+            if (!$this->sHouseNumber && $this->iOperator != Operator::POSTCODE) {
+                $oSearch = clone $this;
+                $oSearch->iSearchRank++;
+                $oSearch->iNamePhrase = -1;
+                $oSearch->sHouseNumber = $oSearchTerm->sToken;
+                if ($this->iOperator != Operator::NONE) {
+                    $oSearch->iSearchRank++;
+                }
+                // sanity check: if the housenumber is not mainly made
+                // up of numbers, add a penalty
+                if (preg_match('/\\d/', $oSearch->sHouseNumber) === 0
+                    || preg_match_all('/[^0-9]/', $oSearch->sHouseNumber, $aMatches) > 2) {
+                    $oSearch->iSearchRank++;
+                }
+                if (empty($oSearchTerm->iId)) {
+                    $oSearch->iSearchRank++;
+                }
+                // also must not appear in the middle of the address
+                if (!empty($this->aAddress)
+                    || (!empty($this->aAddressNonSearch))
+                    || $this->sPostcode
+                ) {
+                    $oSearch->iSearchRank++;
+                }
+                $aNewSearches[] = $oSearch;
+                // Housenumbers may appear in the name when the place has its own
+                // address terms.
+                if ($oSearchTerm->iId !== null
+                    && ($this->iNamePhrase >= 0 || empty($this->aName))
+                    && empty($this->aAddress)
+                   ) {
+                    $oSearch = clone $this;
+                    $oSearch->iSearchRank++;
+                    $oSearch->aAddress = $this->aName;
+                    $oSearch->bRareName = false;
+                    $oSearch->aName = array($oSearchTerm->iId => $oSearchTerm->iId);
+                    $aNewSearches[] = $oSearch;
+                }
+            }
+        } elseif ($sPhraseType == ''
+                  && is_a($oSearchTerm, '\Nominatim\Token\SpecialTerm')
+        ) {
+            if ($this->iOperator == Operator::NONE) {
+                $oSearch = clone $this;
+                $oSearch->iSearchRank += 2;
+                $oSearch->iNamePhrase = -1;
 
-    /**
-     * Check if a postcode is currently included in the search.
-     *
-     * @return bool True, if a postcode is set.
-     */
-    public function hasPostcode()
-    {
-        return $this->sPostcode !== '';
-    }
+                $iOp = $oSearchTerm->iOperator;
+                if ($iOp == Operator::NONE) {
+                    if (!empty($this->aName) || $this->oContext->isBoundedSearch()) {
+                        $iOp = Operator::NAME;
+                    } else {
+                        $iOp = Operator::NEAR;
+                    }
+                    $oSearch->iSearchRank += 2;
+                } elseif (!$bFirstToken && !$bLastToken) {
+                    $oSearch->iSearchRank += 2;
+                }
+                if ($this->sHouseNumber) {
+                    $oSearch->iSearchRank++;
+                }
 
-    /**
-     * Check if a house number is set for the search.
-     *
-     * @return bool True, if a house number is set.
-     */
-    public function hasHousenumber()
-    {
-        return $this->sHouseNumber !== '';
-    }
-
-    /**
-     * Check if a special type of place is requested.
-     *
-     * param integer iOperator  When set, check for the particular
-     *                          operator used for the special type.
-     *
-     * @return bool True, if speial type is requested or, if requested,
-     *              a special type with the given operator.
-     */
-    public function hasOperator($iOperator = null)
-    {
-        return $iOperator === null ? $this->iOperator != Operator::NONE : $this->iOperator == $iOperator;
-    }
-
-    /**
-     * Add the given token to the list of terms to search for in the address.
-     *
-     * @param integer iID       ID of term to add.
-     * @param bool bSearchable  Term should be used to search for result
-     *                          (i.e. term is not a stop word).
-     */
-    public function addAddressToken($iId, $bSearchable = true)
-    {
-        if ($bSearchable) {
-            $this->aAddress[$iId] = $iId;
-        } else {
-            $this->aAddressNonSearch[$iId] = $iId;
+                $oSearch->setPoiSearch(
+                    $iOp,
+                    $oSearchTerm->sClass,
+                    $oSearchTerm->sType
+                );
+                $aNewSearches[] = $oSearch;
+            }
+        } elseif ($sPhraseType != 'country'
+                  && is_a($oSearchTerm, '\Nominatim\Token\Word')
+        ) {
+            $iWordID = $oSearchTerm->iId;
+            // Full words can only be a name if they appear at the beginning
+            // of the phrase. In structured search the name must forcably in
+            // the first phrase. In unstructured search it may be in a later
+            // phrase when the first phrase is a house number.
+            if (!empty($this->aName) || !($bFirstPhrase || $sPhraseType == '')) {
+                if (($sPhraseType == '' || !$bFirstPhrase) && !$bHasPartial) {
+                    $oSearch = clone $this;
+                    $oSearch->iNamePhrase = -1;
+                    $oSearch->iSearchRank += 3 * $oSearchTerm->iTermCount;
+                    $oSearch->aAddress[$iWordID] = $iWordID;
+                    $aNewSearches[] = $oSearch;
+                }
+            } elseif (empty($this->aNameNonSearch)) {
+                $oSearch = clone $this;
+                $oSearch->iSearchRank++;
+                $oSearch->aName = array($iWordID => $iWordID);
+                if (CONST_Search_NameOnlySearchFrequencyThreshold) {
+                    $oSearch->bRareName =
+                        $oSearchTerm->iSearchNameCount
+                          < CONST_Search_NameOnlySearchFrequencyThreshold;
+                }
+                $aNewSearches[] = $oSearch;
+            }
         }
+
+        return $aNewSearches;
     }
 
     /**
-     * Add the given full-word token to the list of terms to search for in the
-     * name.
+     * Derive new searches by adding a partial term to the existing search.
      *
-     * @param interger iId    ID of term to add.
-     * @param bool bRareName  True if the term is infrequent enough to not
-     *                        require other constraints for efficient search.
-     */
-    public function addNameToken($iId, $bRareName)
-    {
-        $this->aName[$iId] = $iId;
-        $this->bRareName = $bRareName;
-    }
-
-    /**
-     * Add the given partial token to the list of terms to search for in
-     * the name.
+     * @param string  $sToken             Term for the token.
+     * @param object  $oSearchTerm        Description of the token.
+     * @param bool    $bStructuredPhrases True if the search is structured.
+     * @param integer $iPhrase            Number of the phrase the token is in.
+     * @param array[] $aFullTokens        List of full term tokens with the
+     *                                    same name.
      *
-     * @param integer iID            ID of term to add.
-     * @param bool bSearchable       Term should be used to search for result
-     *                               (i.e. term is not a stop word).
-     * @param integer iPhraseNumber  Index of phrase, where the partial term
-     *                               appears.
+     * @return SearchDescription[] List of derived search descriptions.
      */
-    public function addPartialNameToken($iId, $bSearchable, $iPhraseNumber)
+    public function extendWithPartialTerm($sToken, $oSearchTerm, $bStructuredPhrases, $iPhrase, $aFullTokens)
     {
-        if ($bSearchable) {
-            $this->aName[$iId] = $iId;
-        } else {
-            $this->aNameNonSearch[$iId] = $iId;
+        // Only allow name terms.
+        if (!(is_a($oSearchTerm, '\Nominatim\Token\Word'))) {
+            return array();
         }
-        $this->iNamePhrase = $iPhraseNumber;
-    }
 
-    /**
-     * Set country restriction for the search.
-     *
-     * @param string sCountryCode  Country code of country to restrict search to.
-     */
-    public function setCountry($sCountryCode)
-    {
-        $this->sCountryCode = $sCountryCode;
-        $this->iNamePhrase = -1;
-    }
+        $aNewSearches = array();
+        $iWordID = $oSearchTerm->iId;
 
-    /**
-     * Set postcode search constraint.
-     *
-     * @param string sPostcode  Postcode the result should have.
-     */
-    public function setPostcode($sPostcode)
-    {
-        $this->sPostcode = $sPostcode;
-        $this->iNamePhrase = -1;
-    }
+        if ((!$bStructuredPhrases || $iPhrase > 0)
+            && (!empty($this->aName))
+        ) {
+            $oSearch = clone $this;
+            $oSearch->iSearchRank++;
+            if (preg_match('#^[0-9 ]+$#', $sToken)) {
+                $oSearch->iSearchRank++;
+            }
+            if ($oSearchTerm->iSearchNameCount < CONST_Max_Word_Frequency) {
+                $oSearch->aAddress[$iWordID] = $iWordID;
+            } else {
+                $oSearch->aAddressNonSearch[$iWordID] = $iWordID;
+                if (!empty($aFullTokens)) {
+                    $oSearch->iSearchRank++;
+                }
+            }
+            $aNewSearches[] = $oSearch;
+        }
 
-    /**
-     * Make this search a search for a postcode object.
-     *
-     * @param integer iId       Token Id for the postcode.
-     * @param string sPostcode  Postcode to look for.
-     */
-    public function setPostcodeAsName($iId, $sPostcode)
-    {
-        $this->iOperator = Operator::POSTCODE;
-        $this->aAddress = array_merge($this->aAddress, $this->aName);
-        $this->aName = array($iId => $sPostcode);
-        $this->bRareName = true;
-        $this->iNamePhrase = -1;
-    }
+        if ((!$this->sPostcode && !$this->aAddress && !$this->aAddressNonSearch)
+            && ((empty($this->aName) && empty($this->aNameNonSearch)) || $this->iNamePhrase == $iPhrase)
+            && strpos($sToken, ' ') === false
+        ) {
+            $oSearch = clone $this;
+            $oSearch->iSearchRank++;
+            if (empty($this->aName) && empty($this->aNameNonSearch)) {
+                $oSearch->iSearchRank++;
+            }
+            if (preg_match('#^[0-9 ]+$#', $sToken)) {
+                $oSearch->iSearchRank++;
+            }
+            if ($oSearchTerm->iSearchNameCount < CONST_Max_Word_Frequency) {
+                if (empty($this->aName)
+                    && CONST_Search_NameOnlySearchFrequencyThreshold
+                ) {
+                    $oSearch->bRareName =
+                        $oSearchTerm->iSearchNameCount
+                          < CONST_Search_NameOnlySearchFrequencyThreshold;
+                } else {
+                    $oSearch->bRareName = false;
+                }
+                $oSearch->aName[$iWordID] = $iWordID;
+            } else {
+                if (!empty($aFullTokens)) {
+                    $oSearch->iSearchRank++;
+                }
+                $oSearch->aNameNonSearch[$iWordID] = $iWordID;
+            }
+            $oSearch->iNamePhrase = $iPhrase;
+            $aNewSearches[] = $oSearch;
+        }
 
-    /**
-     * Set house number search cnstraint.
-     *
-     * @param string sNumber  House number the result should have.
-     */
-    public function setHousenumber($sNumber)
-    {
-        $this->sHouseNumber = $sNumber;
-        $this->iNamePhrase = -1;
-    }
-
-    /**
-     * Make this search a search for a house number.
-     *
-     * @param integer iId  Token Id for the house number.
-     */
-    public function setHousenumberAsName($iId)
-    {
-        $this->aAddress = array_merge($this->aAddress, $this->aName);
-        $this->bRareName = false;
-        $this->aName = array($iId => $iId);
-        $this->iNamePhrase = -1;
-    }
-
-    /**
-     * Make this search a POI search.
-     *
-     * In a POI search, objects are not (only) searched by their name
-     * but also by the primary OSM key/value pair (class and type in Nominatim).
-     *
-     * @param integer $iOperator Type of POI search
-     * @param string  $sClass    Class (or OSM tag key) of POI.
-     * @param string  $sType     Type (or OSM tag value) of POI.
-     *
-     * @return void
-     */
-    public function setPoiSearch($iOperator, $sClass, $sType)
-    {
-        $this->iOperator = $iOperator;
-        $this->sClass = $sClass;
-        $this->sType = $sType;
-        $this->iNamePhrase = -1;
-    }
-
-    public function getNamePhrase()
-    {
-        return $this->iNamePhrase;
-    }
-
-    /**
-     * Get the global search context.
-     *
-     * @return object  Objects of global search constraints.
-     */
-    public function getContext()
-    {
-        return $this->oContext;
+        return $aNewSearches;
     }
 
     /////////// Query functions
@@ -367,6 +413,7 @@ class SearchDescription
     public function query(&$oDB, $iMinRank, $iMaxRank, $iLimit)
     {
         $aResults = array();
+        $iHousenumber = -1;
 
         if ($this->sCountryCode
             && empty($this->aName)
@@ -399,24 +446,23 @@ class SearchDescription
 
             // Now search for housenumber, if housenumber provided. Can be zero.
             if (($this->sHouseNumber || $this->sHouseNumber === '0') && !empty($aResults)) {
-                $aHnResults = $this->queryHouseNumber($oDB, $aResults);
-
                 // Downgrade the rank of the street results, they are missing
-                // the housenumber. Also drop POI places (rank 30) here, they
-                // cannot be a parent place and therefore must not be shown
-                // as a result for a search with a missing housenumber.
+                // the housenumber.
                 foreach ($aResults as $oRes) {
-                    if ($oRes->iAddressRank < 28) {
-                        if ($oRes->iAddressRank >= 26) {
-                            $oRes->iResultRank++;
-                        } else {
-                            $oRes->iResultRank += 2;
-                        }
-                        $aHnResults[$oRes->iId] = $oRes;
+                    if ($oRes->iAddressRank >= 26) {
+                        $oRes->iResultRank++;
+                    } else {
+                        $oRes->iResultRank += 2;
                     }
                 }
 
-                $aResults = $aHnResults;
+                $aHnResults = $this->queryHouseNumber($oDB, $aResults);
+
+                if (!empty($aHnResults)) {
+                    foreach ($aHnResults as $oRes) {
+                        $aResults[$oRes->iId] = $oRes;
+                    }
+                }
             }
 
             // finally get POIs if requested
@@ -696,33 +742,16 @@ class SearchDescription
     private function queryHouseNumber(&$oDB, $aRoadPlaceIDs)
     {
         $aResults = array();
-        $sRoadPlaceIDs = Result::joinIdsByTableMaxRank(
-            $aRoadPlaceIDs,
-            Result::TABLE_PLACEX,
-            27
-        );
-        $sPOIPlaceIDs = Result::joinIdsByTableMinRank(
-            $aRoadPlaceIDs,
-            Result::TABLE_PLACEX,
-            30
-        );
+        $sPlaceIDs = Result::joinIdsByTable($aRoadPlaceIDs, Result::TABLE_PLACEX);
 
-        $aIDCondition = array();
-        if ($sRoadPlaceIDs) {
-            $aIDCondition[] = 'parent_place_id in ('.$sRoadPlaceIDs.')';
-        }
-        if ($sPOIPlaceIDs) {
-            $aIDCondition[] = 'place_id in ('.$sPOIPlaceIDs.')';
-        }
-
-        if (empty($aIDCondition)) {
+        if (!$sPlaceIDs) {
             return $aResults;
         }
 
         $sHouseNumberRegex = '\\\\m'.$this->sHouseNumber.'\\\\M';
-        $sSQL = 'SELECT place_id FROM placex WHERE';
-        $sSQL .= "  housenumber ~* E'".$sHouseNumberRegex."'";
-        $sSQL .= ' AND ('.join(' OR ', $aIDCondition).')';
+        $sSQL = 'SELECT place_id FROM placex ';
+        $sSQL .= 'WHERE parent_place_id in ('.$sPlaceIDs.')';
+        $sSQL .= "  AND housenumber ~* E'".$sHouseNumberRegex."'";
         $sSQL .= $this->oContext->excludeSQL(' AND place_id');
 
         Debug::printSQL($sSQL);
@@ -734,11 +763,11 @@ class SearchDescription
 
         $bIsIntHouseNumber= (bool) preg_match('/[0-9]+/', $this->sHouseNumber);
         $iHousenumber = intval($this->sHouseNumber);
-        if ($bIsIntHouseNumber && $sRoadPlaceIDs && empty($aResults)) {
+        if ($bIsIntHouseNumber && empty($aResults)) {
             // if nothing found, search in the interpolation line table
             $sSQL = 'SELECT distinct place_id FROM location_property_osmline';
             $sSQL .= ' WHERE startnumber is not NULL';
-            $sSQL .= '  AND parent_place_id in ('.$sRoadPlaceIDs.') AND (';
+            $sSQL .= '  AND parent_place_id in ('.$sPlaceIDs.') AND (';
             if ($iHousenumber % 2 == 0) {
                 // If housenumber is even, look for housenumber in streets
                 // with interpolationtype even or all.
@@ -761,10 +790,24 @@ class SearchDescription
             }
         }
 
+        // If nothing found try the aux fallback table
+        if (CONST_Use_Aux_Location_data && empty($aResults)) {
+            $sSQL = 'SELECT place_id FROM location_property_aux';
+            $sSQL .= ' WHERE parent_place_id in ('.$sPlaceIDs.')';
+            $sSQL .= " AND housenumber = '".$this->sHouseNumber."'";
+            $sSQL .= $this->oContext->excludeSQL(' AND place_id');
+
+            Debug::printSQL($sSQL);
+
+            foreach ($oDB->getCol($sSQL) as $iPlaceId) {
+                $aResults[$iPlaceId] = new Result($iPlaceId, Result::TABLE_AUX);
+            }
+        }
+
         // If nothing found then search in Tiger data (location_property_tiger)
-        if (CONST_Use_US_Tiger_Data && $sRoadPlaceIDs && $bIsIntHouseNumber && empty($aResults)) {
+        if (CONST_Use_US_Tiger_Data && $bIsIntHouseNumber && empty($aResults)) {
             $sSQL = 'SELECT place_id FROM location_property_tiger';
-            $sSQL .= ' WHERE parent_place_id in ('.$sRoadPlaceIDs.') and (';
+            $sSQL .= ' WHERE parent_place_id in ('.$sPlaceIDs.') and (';
             if ($iHousenumber % 2 == 0) {
                 $sSQL .= "interpolationtype='even'";
             } else {

lib-php/SearchPosition.php

@@ -1,87 +0,0 @@
-<?php
-
-namespace Nominatim;
-
-/**
- * Description of the position of a token within a query.
- */
-class SearchPosition
-{
-    private $sPhraseType;
-
-    private $iPhrase;
-    private $iNumPhrases;
-
-    private $iToken;
-    private $iNumTokens;
-
-
-    public function __construct($sPhraseType, $iPhrase, $iNumPhrases)
-    {
-        $this->sPhraseType = $sPhraseType;
-        $this->iPhrase = $iPhrase;
-        $this->iNumPhrases = $iNumPhrases;
-    }
-
-    public function setTokenPosition($iToken, $iNumTokens)
-    {
-        $this->iToken = $iToken;
-        $this->iNumTokens = $iNumTokens;
-    }
-
-    /**
-     * Check if the phrase can be of the given type.
-     *
-     * @param string  $sType  Type of phrse requested.
-     *
-     * @return True if the phrase is untyped or of the given type.
-     */
-    public function maybePhrase($sType)
-    {
-        return $this->sPhraseType == '' || $this->sPhraseType == $sType;
-    }
-
-    /**
-     * Check if the phrase is exactly of the given type.
-     *
-     * @param string  $sType  Type of phrse requested.
-     *
-     * @return True if the phrase of the given type.
-     */
-    public function isPhrase($sType)
-    {
-        return $this->sPhraseType == $sType;
-    }
-
-    /**
-     * Return true if the token is the very first in the query.
-     */
-    public function isFirstToken()
-    {
-        return $this->iPhrase == 0 && $this->iToken == 0;
-    }
-
-    /**
-     * Check if the token is the final one in the query.
-     */
-    public function isLastToken()
-    {
-        return $this->iToken + 1 == $this->iNumTokens && $this->iPhrase + 1 == $this->iNumPhrases;
-    }
-
-    /**
-     * Check if the current token is part of the first phrase in the query.
-     */
-    public function isFirstPhrase()
-    {
-        return $this->iPhrase == 0;
-    }
-
-    /**
-     * Get the phrase position in the query.
-     */
-    public function getPhrase()
-    {
-        return $this->iPhrase;
-    }
-}

lib-php/Shell.php

@@ -33,9 +33,7 @@ class Shell
     public function addEnvPair($sKey, $sVal)
     {
         if (isset($sKey) && $sKey && isset($sVal)) {
-            if (!isset($this->aEnv)) {
-                $this->aEnv = $_ENV;
-            }
+            if (!isset($this->aEnv)) $this->aEnv = $_ENV;
             $this->aEnv = array_merge($this->aEnv, array($sKey => $sVal), $_ENV);
         }
         return $this;
@@ -77,8 +75,11 @@ class Shell
         return $iStat;
     }
 
+
+
     private function escapeParam($sParam)
     {
-        return (preg_match('/^-*\w+$/', $sParam)) ? $sParam : escapeshellarg($sParam);
+        if (preg_match('/^-*\w+$/', $sParam)) return $sParam;
+        return escapeshellarg($sParam);
     }
 }

lib-php/Status.php

@@ -2,8 +2,6 @@
 
 namespace Nominatim;
 
-require_once(CONST_TokenizerDir.'/tokenizer.php');
-
 use Exception;
 
 class Status
@@ -27,8 +25,24 @@ class Status
             throw new Exception('Database connection failed', 700);
         }
 
-        $oTokenizer = new \Nominatim\Tokenizer($this->oDB);
-        $oTokenizer->checkStatus();
+        $sStandardWord = $this->oDB->getOne("SELECT make_standard_name('a')");
+        if ($sStandardWord === false) {
+            throw new Exception('Module failed', 701);
+        }
+
+        if ($sStandardWord != 'a') {
+            throw new Exception('Module call failed', 702);
+        }
+
+        $sSQL = 'SELECT word_id, word_token, word, class, type, country_code, ';
+        $sSQL .= "operator, search_name_count FROM word WHERE word_token IN (' a')";
+        $iWordID = $this->oDB->getOne($sSQL);
+        if ($iWordID === false) {
+            throw new Exception('Query failed', 703);
+        }
+        if (!$iWordID) {
+            throw new Exception('No value', 704);
+        }
     }
 
     public function dataDate()

lib-php/TokenCountry.php

@@ -8,9 +8,9 @@ namespace Nominatim\Token;
 class Country
 {
     /// Database word id, if available.
-    private $iId;
+    public $iId;
     /// Two-letter country code (lower-cased).
-    private $sCountryCode;
+    public $sCountryCode;
 
     public function __construct($iId, $sCountryCode)
     {
@@ -18,44 +18,6 @@ class Country
         $this->sCountryCode = $sCountryCode;
     }
 
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oSearch->hasCountry() && $oPosition->maybePhrase('country');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        $oNewSearch = $oSearch->clone($oPosition->isLastToken() ? 1 : 6);
-        $oNewSearch->setCountry($this->sCountryCode);
-
-        return array($oNewSearch);
-    }
-
     public function debugInfo()
     {
         return array(
@@ -64,9 +26,4 @@ class Country
                 'Info' => $this->sCountryCode
                );
     }
-
-    public function debugCode()
-    {
-        return 'C';
-    }
 }

lib-php/TokenHousenumber.php

@@ -8,9 +8,9 @@ namespace Nominatim\Token;
 class HouseNumber
 {
     /// Database word id, if available.
-    private $iId;
+    public $iId;
     /// Normalized house number.
-    private $sToken;
+    public $sToken;
 
     public function __construct($iId, $sToken)
     {
@@ -18,80 +18,6 @@ class HouseNumber
         $this->sToken = $sToken;
     }
 
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oSearch->hasHousenumber()
-               && !$oSearch->hasOperator(\Nominatim\Operator::POSTCODE)
-               && $oPosition->maybePhrase('street');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        $aNewSearches = array();
-
-        // sanity check: if the housenumber is not mainly made
-        // up of numbers, add a penalty
-        $iSearchCost = 1;
-        if (preg_match('/\\d/', $this->sToken) === 0
-            || preg_match_all('/[^0-9]/', $this->sToken, $aMatches) > 2) {
-            $iSearchCost++;
-        }
-        if (!$oSearch->hasOperator(\Nominatim\Operator::NONE)) {
-            $iSearchCost++;
-        }
-        if (empty($this->iId)) {
-            $iSearchCost++;
-        }
-        // also must not appear in the middle of the address
-        if ($oSearch->hasAddress() || $oSearch->hasPostcode()) {
-            $iSearchCost++;
-        }
-
-        $oNewSearch = $oSearch->clone($iSearchCost);
-        $oNewSearch->setHousenumber($this->sToken);
-        $aNewSearches[] = $oNewSearch;
-
-        // Housenumbers may appear in the name when the place has its own
-        // address terms.
-        if ($this->iId !== null
-            && ($oSearch->getNamePhrase() >= 0 || !$oSearch->hasName())
-            && !$oSearch->hasAddress()
-        ) {
-            $oNewSearch = $oSearch->clone($iSearchCost);
-            $oNewSearch->setHousenumberAsName($this->iId);
-
-            $aNewSearches[] = $oNewSearch;
-        }
-
-        return $aNewSearches;
-    }
-
-
     public function debugInfo()
     {
         return array(
@@ -100,9 +26,4 @@ class HouseNumber
                 'Info' => array('nr' => $this->sToken)
                );
     }
-
-    public function debugCode()
-    {
-        return 'H';
-    }
 }

lib-php/TokenList.php

@@ -7,7 +7,6 @@ require_once(CONST_LibDir.'/TokenHousenumber.php');
 require_once(CONST_LibDir.'/TokenPostcode.php');
 require_once(CONST_LibDir.'/TokenSpecialTerm.php');
 require_once(CONST_LibDir.'/TokenWord.php');
-require_once(CONST_LibDir.'/TokenPartial.php');
 require_once(CONST_LibDir.'/SpecialSearchOperator.php');
 
 /**
@@ -18,6 +17,15 @@ require_once(CONST_LibDir.'/SpecialSearchOperator.php');
  * tokens do not have a common base class. All tokens need to have a field
  * with the word id that points to an entry in the `word` database table
  * but otherwise the information saved about a token can be very different.
+ *
+ * There are two different kinds of token words: full words and partial terms.
+ *
+ * Full words start with a space. They represent a complete name of a place.
+ * All special tokens are normally full words.
+ *
+ * Partial terms have no space at the beginning. They may represent a part of
+ * a name of a place (e.g. in the name 'World Trade Center' a partial term
+ * would be 'Trade' or 'Trade Center'). They are only used in TokenWord.
  */
 class TokenList
 {
@@ -56,7 +64,7 @@ class TokenList
      */
     public function containsAny($sWord)
     {
-        return isset($this->aTokens[$sWord]);
+        return isset($this->aTokens[$sWord]) || isset($this->aTokens[' '.$sWord]);
     }
 
     /**
@@ -78,8 +86,8 @@ class TokenList
 
         foreach ($this->aTokens as $aTokenList) {
             foreach ($aTokenList as $oToken) {
-                if (is_a($oToken, '\Nominatim\Token\Word')) {
-                    $ids[$oToken->getId()] = $oToken->getId();
+                if (is_a($oToken, '\Nominatim\Token\Word') && !$oToken->bPartial) {
+                    $ids[$oToken->iId] = $oToken->iId;
                 }
             }
         }
@@ -87,6 +95,88 @@ class TokenList
         return $ids;
     }
 
+    /**
+     * Add token information from the word table in the database.
+     *
+     * @param object   $oDB           Nominatim::DB instance.
+     * @param string[] $aTokens       List of tokens to look up in the database.
+     * @param string[] $aCountryCodes List of country restrictions.
+     * @param string   $sNormQuery    Normalized query string.
+     * @param object   $oNormalizer   Normalizer function to use on tokens.
+     *
+     * @return void
+     */
+    public function addTokensFromDB(&$oDB, &$aTokens, &$aCountryCodes, $sNormQuery, $oNormalizer)
+    {
+        // Check which tokens we have, get the ID numbers
+        $sSQL = 'SELECT word_id, word_token, word, class, type, country_code,';
+        $sSQL .= ' operator, coalesce(search_name_count, 0) as count';
+        $sSQL .= ' FROM word WHERE word_token in (';
+        $sSQL .= join(',', $oDB->getDBQuotedList($aTokens)).')';
+
+        Debug::printSQL($sSQL);
+
+        $aDBWords = $oDB->getAll($sSQL, null, 'Could not get word tokens.');
+
+        foreach ($aDBWords as $aWord) {
+            $oToken = null;
+            $iId = (int) $aWord['word_id'];
+
+            if ($aWord['class']) {
+                // Special terms need to appear in their normalized form.
+                if ($aWord['word']) {
+                    $sNormWord = $aWord['word'];
+                    if ($oNormalizer != null) {
+                        $sNormWord = $oNormalizer->transliterate($aWord['word']);
+                    }
+                    if (strpos($sNormQuery, $sNormWord) === false) {
+                        continue;
+                    }
+                }
+
+                if ($aWord['class'] == 'place' && $aWord['type'] == 'house') {
+                    $oToken = new Token\HouseNumber($iId, trim($aWord['word_token']));
+                } elseif ($aWord['class'] == 'place' && $aWord['type'] == 'postcode') {
+                    if ($aWord['word']
+                        && pg_escape_string($aWord['word']) == $aWord['word']
+                    ) {
+                        $oToken = new Token\Postcode(
+                            $iId,
+                            $aWord['word'],
+                            $aWord['country_code']
+                        );
+                    }
+                } else {
+                    // near and in operator the same at the moment
+                    $oToken = new Token\SpecialTerm(
+                        $iId,
+                        $aWord['class'],
+                        $aWord['type'],
+                        $aWord['operator'] ? Operator::NEAR : Operator::NONE
+                    );
+                }
+            } elseif ($aWord['country_code']) {
+                // Filter country tokens that do not match restricted countries.
+                if (!$aCountryCodes
+                    || in_array($aWord['country_code'], $aCountryCodes)
+                ) {
+                    $oToken = new Token\Country($iId, $aWord['country_code']);
+                }
+            } else {
+                $oToken = new Token\Word(
+                    $iId,
+                    $aWord['word_token'][0] != ' ',
+                    (int) $aWord['count'],
+                    substr_count($aWord['word_token'], ' ')
+                );
+            }
+
+            if ($oToken) {
+                $this->addToken($aWord['word_token'], $oToken);
+            }
+        }
+    }
+
     /**
      * Add a new token for the given word.
      *
@@ -109,9 +199,9 @@ class TokenList
         $aWordsIDs = array();
         foreach ($this->aTokens as $sToken => $aWords) {
             foreach ($aWords as $aToken) {
-                $iId = $aToken->getId();
-                if ($iId !== null) {
-                    $aWordsIDs[$iId] = '#'.$sToken.'('.$aToken->debugCode().' '.$iId.')#';
+                if ($aToken->iId !== null) {
+                    $aWordsIDs[$aToken->iId] =
+                        '#'.$sToken.'('.$aToken->iId.')#';
                 }
             }
         }

lib-php/TokenPartial.php

@@ -1,118 +0,0 @@
-<?php
-
-namespace Nominatim\Token;
-
-/**
- * A standard word token.
- */
-class Partial
-{
-    /// Database word id, if applicable.
-    private $iId;
-    /// Number of appearances in the database.
-    private $iSearchNameCount;
-    /// True, if the token consists exclusively of digits and spaces.
-    private $bNumberToken;
-
-    public function __construct($iId, $sToken, $iSearchNameCount)
-    {
-        $this->iId = $iId;
-        $this->bNumberToken = (bool) preg_match('#^[0-9 ]+$#', $sToken);
-        $this->iSearchNameCount = $iSearchNameCount;
-    }
-
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oPosition->isPhrase('country');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        $aNewSearches = array();
-
-        // Partial token in Address.
-        if (($oPosition->isPhrase('') || !$oPosition->isFirstPhrase())
-            && $oSearch->hasName()
-        ) {
-            $iSearchCost = $this->bNumberToken ? 2 : 1;
-            if ($this->iSearchNameCount >= CONST_Max_Word_Frequency) {
-                $iSearchCost += 1;
-            }
-
-            $oNewSearch = $oSearch->clone($iSearchCost);
-            $oNewSearch->addAddressToken(
-                $this->iId,
-                $this->iSearchNameCount < CONST_Max_Word_Frequency
-            );
-
-            $aNewSearches[] = $oNewSearch;
-        }
-
-        // Partial token in Name.
-        if ((!$oSearch->hasPostcode() && !$oSearch->hasAddress())
-            && (!$oSearch->hasName(true)
-                || $oSearch->getNamePhrase() == $oPosition->getPhrase())
-        ) {
-            $iSearchCost = 1;
-            if (!$oSearch->hasName(true)) {
-                $iSearchCost += 1;
-            }
-            if ($this->bNumberToken) {
-                $iSearchCost += 1;
-            }
-
-            $oNewSearch = $oSearch->clone($iSearchCost);
-            $oNewSearch->addPartialNameToken(
-                $this->iId,
-                $this->iSearchNameCount < CONST_Max_Word_Frequency,
-                $oPosition->getPhrase()
-            );
-
-            $aNewSearches[] = $oNewSearch;
-        }
-
-        return $aNewSearches;
-    }
-
-
-    public function debugInfo()
-    {
-        return array(
-                'ID' => $this->iId,
-                'Type' => 'partial',
-                'Info' => array(
-                           'count' => $this->iSearchNameCount
-                          )
-               );
-    }
-
-    public function debugCode()
-    {
-        return 'w';
-    }
-}

lib-php/TokenPostcode.php

@@ -8,11 +8,11 @@ namespace Nominatim\Token;
 class Postcode
 {
     /// Database word id, if available.
-    private $iId;
+    public $iId;
     /// Full nomralized postcode (upper cased).
-    private $sPostcode;
+    public $sPostcode;
     // Optional country code the postcode belongs to (currently unused).
-    private $sCountryCode;
+    public $sCountryCode;
 
     public function __construct($iId, $sPostcode, $sCountryCode = '')
     {
@@ -21,67 +21,6 @@ class Postcode
         $this->sCountryCode = empty($sCountryCode) ? '' : $sCountryCode;
     }
 
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oSearch->hasPostcode() && $oPosition->maybePhrase('postalcode');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        $aNewSearches = array();
-
-        // If we have structured search or this is the first term,
-        // make the postcode the primary search element.
-        if ($oSearch->hasOperator(\Nominatim\Operator::NONE) && $oPosition->isFirstToken()) {
-            $oNewSearch = $oSearch->clone(1);
-            $oNewSearch->setPostcodeAsName($this->iId, $this->sPostcode);
-
-            $aNewSearches[] = $oNewSearch;
-        }
-
-        // If we have a structured search or this is not the first term,
-        // add the postcode as an addendum.
-        if (!$oSearch->hasOperator(\Nominatim\Operator::POSTCODE)
-            && ($oPosition->isPhrase('postalcode') || $oSearch->hasName())
-        ) {
-            $iPenalty = 1;
-            if (strlen($this->sPostcode) < 4) {
-                $iPenalty += 4 - strlen($this->sPostcode);
-            }
-            $oNewSearch = $oSearch->clone($iPenalty);
-            $oNewSearch->setPostcode($this->sPostcode);
-
-            $aNewSearches[] = $oNewSearch;
-        }
-
-        return $aNewSearches;
-    }
-
     public function debugInfo()
     {
         return array(
@@ -90,9 +29,4 @@ class Postcode
                 'Info' => $this->sPostcode.'('.$this->sCountryCode.')'
                );
     }
-
-    public function debugCode()
-    {
-        return 'P';
-    }
 }

lib-php/TokenSpecialTerm.php

@@ -10,13 +10,13 @@ require_once(CONST_LibDir.'/SpecialSearchOperator.php');
 class SpecialTerm
 {
     /// Database word id, if applicable.
-    private $iId;
+    public $iId;
     /// Class (or OSM tag key) of the place to look for.
-    private $sClass;
+    public $sClass;
     /// Type (or OSM tag value) of the place to look for.
-    private $sType;
+    public $sType;
     /// Relationship of the operator to the object (see Operator class).
-    private $iOperator;
+    public $iOperator;
 
     public function __construct($iID, $sClass, $sType, $iOperator)
     {
@@ -26,62 +26,6 @@ class SpecialTerm
         $this->iOperator = $iOperator;
     }
 
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oSearch->hasOperator() && $oPosition->isPhrase('');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        $iSearchCost = 2;
-
-        $iOp = $this->iOperator;
-        if ($iOp == \Nominatim\Operator::NONE) {
-            if ($oSearch->hasName() || $oSearch->getContext()->isBoundedSearch()) {
-                $iOp = \Nominatim\Operator::NAME;
-            } else {
-                $iOp = \Nominatim\Operator::NEAR;
-            }
-            $iSearchCost += 2;
-        } elseif (!$oPosition->isFirstToken() && !$oPosition->isLastToken()) {
-            $iSearchCost += 2;
-        }
-        if ($oSearch->hasHousenumber()) {
-            $iSearchCost ++;
-        }
-
-        $oNewSearch = $oSearch->clone($iSearchCost);
-        $oNewSearch->setPoiSearch($iOp, $this->sClass, $this->sType);
-
-        return array($oNewSearch);
-    }
-
-
     public function debugInfo()
     {
         return array(
@@ -94,9 +38,4 @@ class SpecialTerm
                           )
                );
     }
-
-    public function debugCode()
-    {
-        return 'S';
-    }
 }

lib-php/TokenWord.php

@@ -8,95 +8,31 @@ namespace Nominatim\Token;
 class Word
 {
     /// Database word id, if applicable.
-    private $iId;
+    public $iId;
+    /// If true, the word may represent only part of a place name.
+    public $bPartial;
     /// Number of appearances in the database.
-    private $iSearchNameCount;
+    public $iSearchNameCount;
     /// Number of terms in the word.
-    private $iTermCount;
+    public $iTermCount;
 
-    public function __construct($iId, $iSearchNameCount, $iTermCount)
+    public function __construct($iId, $bPartial, $iSearchNameCount, $iTermCount)
     {
         $this->iId = $iId;
+        $this->bPartial = $bPartial;
         $this->iSearchNameCount = $iSearchNameCount;
         $this->iTermCount = $iTermCount;
     }
 
-    public function getId()
-    {
-        return $this->iId;
-    }
-
-    /**
-     * Check if the token can be added to the given search.
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return True if the token is compatible with the search configuration
-     *         given the position.
-     */
-    public function isExtendable($oSearch, $oPosition)
-    {
-        return !$oPosition->isPhrase('country');
-    }
-
-    /**
-     * Derive new searches by adding this token to an existing search.
-     *
-     * @param object  $oSearch      Partial search description derived so far.
-     * @param object  $oPosition    Description of the token position within
-                                    the query.
-     *
-     * @return SearchDescription[] List of derived search descriptions.
-     */
-    public function extendSearch($oSearch, $oPosition)
-    {
-        // Full words can only be a name if they appear at the beginning
-        // of the phrase. In structured search the name must forcably in
-        // the first phrase. In unstructured search it may be in a later
-        // phrase when the first phrase is a house number.
-        if ($oSearch->hasName()
-            || !($oPosition->isFirstPhrase() || $oPosition->isPhrase(''))
-        ) {
-            if ($this->iTermCount > 1
-                && ($oPosition->isPhrase('') || !$oPosition->isFirstPhrase())
-            ) {
-                $oNewSearch = $oSearch->clone(1);
-                $oNewSearch->addAddressToken($this->iId);
-
-                return array($oNewSearch);
-            }
-        } elseif (!$oSearch->hasName(true)) {
-            $oNewSearch = $oSearch->clone(1);
-            $oNewSearch->addNameToken(
-                $this->iId,
-                CONST_Search_NameOnlySearchFrequencyThreshold
-                && $this->iSearchNameCount
-                          < CONST_Search_NameOnlySearchFrequencyThreshold
-            );
-
-            return array($oNewSearch);
-        }
-
-        return array();
-    }
-
     public function debugInfo()
     {
         return array(
                 'ID' => $this->iId,
                 'Type' => 'word',
                 'Info' => array(
-                           'count' => $this->iSearchNameCount,
-                           'terms' => $this->iTermCount
+                           'partial' => $this->bPartial,
+                           'count' => $this->iSearchNameCount
                           )
                );
     }
-
-    public function debugCode()
-    {
-        return 'W';
-    }
 }

lib-php/admin/check_import_finished.php

@@ -0,0 +1,10 @@
+<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
+
+require_once(CONST_LibDir.'/init-cmd.php');
+
+loadSettings(getcwd());
+
+(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
+    ->addParams('admin', '--check-database')
+    ->run();

lib-php/admin/export.php

@@ -49,9 +49,7 @@
     $oDB->connect();
 
     if (isset($aCMDResult['output-type'])) {
-        if (!isset($aRankmap[$aCMDResult['output-type']])) {
-            fail('unknown output-type: '.$aCMDResult['output-type']);
-        }
+        if (!isset($aRankmap[$aCMDResult['output-type']])) fail('unknown output-type: '.$aCMDResult['output-type']);
         $iOutputRank = $aRankmap[$aCMDResult['output-type']];
     } else {
         $iOutputRank = $aRankmap['street'];
@@ -60,18 +58,14 @@
 
     // Preferred language
     $oParams = new Nominatim\ParameterParser();
-    if (!isset($aCMDResult['language'])) {
-        $aCMDResult['language'] = 'xx';
-    }
+    if (!isset($aCMDResult['language'])) $aCMDResult['language'] = 'xx';
     $aLangPrefOrder = $oParams->getPreferredLanguages($aCMDResult['language']);
     $sLanguagePrefArraySQL = $oDB->getArraySQL($oDB->getDBQuotedList($aLangPrefOrder));
 
     // output formatting: build up a lookup table that maps address ranks to columns
     $aColumnMapping = array();
     $iNumCol = 0;
-    if (!isset($aCMDResult['output-format'])) {
-        $aCMDResult['output-format'] = 'street;suburb;city;county;state;country';
-    }
+    if (!isset($aCMDResult['output-format'])) $aCMDResult['output-format'] = 'street;suburb;city;county;state;country';
     foreach (preg_split('/\s*;\s*/', $aCMDResult['output-format']) as $sColumn) {
         $bHasData = false;
         foreach (preg_split('/\s*,\s*/', $sColumn) as $sRank) {
@@ -86,9 +80,7 @@
                 }
             }
         }
-        if ($bHasData) {
-            $iNumCol++;
-        }
+        if ($bHasData) $iNumCol++;
     }
 
     // build the query for objects
@@ -130,9 +122,7 @@
     if ($sOsmType) {
         $sSQL = 'select place_id from placex where osm_type = :osm_type and osm_id = :osm_id';
         $sParentId = $oDB->getOne($sSQL, array('osm_type' => $sOsmType, 'osm_id' => $sOsmId));
-        if (!$sParentId) {
-            fail('Could not find place '.$sOsmType.' '.$sOsmId);
-        }
+        if (!$sParentId) fail('Could not find place '.$sOsmType.' '.$sOsmId);
     }
     if ($sParentId) {
         $sPlacexSQL .= ' and place_id in (select place_id from place_addressline where address_place_id = '.$sParentId.' and isaddress)';
@@ -146,6 +136,7 @@
     $oResults = $oDB->getQueryStatement($sPlacexSQL);
     $fOutstream = fopen('php://output', 'w');
     while ($aRow = $oResults->fetch()) {
+        //var_dump($aRow);
         $iPlaceID = $aRow['place_id'];
         $sSQL = "select rank_address,get_name_by_language(name,$sLanguagePrefArraySQL) as localname from get_addressdata(:place_id, -1)";
         $sSQL .= ' WHERE isaddress';

lib-php/admin/query.php

@@ -0,0 +1,93 @@
+<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
+
+require_once(CONST_LibDir.'/init-cmd.php');
+require_once(CONST_LibDir.'/Geocode.php');
+require_once(CONST_LibDir.'/ParameterParser.php');
+ini_set('memory_limit', '800M');
+
+$aCMDOptions
+= array(
+   'Query database from command line. Returns search result as JSON.',
+   array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+   array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+   array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+
+   array('search', '', 0, 1, 1, 1, 'string', 'Search for given term or coordinate'),
+   array('country', '', 0, 1, 1, 1, 'string', 'Structured search: country'),
+   array('state', '', 0, 1, 1, 1, 'string', 'Structured search: state'),
+   array('county', '', 0, 1, 1, 1, 'string', 'Structured search: county'),
+   array('city', '', 0, 1, 1, 1, 'string', 'Structured search: city'),
+   array('street', '', 0, 1, 1, 1, 'string', 'Structured search: street'),
+   array('amenity', '', 0, 1, 1, 1, 'string', 'Structured search: amenity'),
+   array('postalcode', '', 0, 1, 1, 1, 'string', 'Structured search: postal code'),
+
+   array('accept-language', '', 0, 1, 1, 1, 'string', 'Preferred language order for showing search results'),
+   array('bounded', '', 0, 1, 0, 0, 'bool', 'Restrict results to given viewbox'),
+   array('nodedupe', '', 0, 1, 0, 0, 'bool', 'Do not remove duplicate results'),
+   array('limit', '', 0, 1, 1, 1, 'int', 'Maximum number of results returned (default: 10)'),
+   array('exclude_place_ids', '', 0, 1, 1, 1, 'string', 'Comma-separated list of place ids to exclude from results'),
+   array('featureType', '', 0, 1, 1, 1, 'string', 'Restrict results to certain features (country, state,city,settlement)'),
+   array('countrycodes', '', 0, 1, 1, 1, 'string', 'Comma-separated list of countries to restrict search to'),
+   array('viewbox', '', 0, 1, 1, 1, 'string', 'Prefer results in given view box'),
+
+   array('project-dir', '', 0, 1, 1, 1, 'realpath', 'Base directory of the Nominatim installation (default: .)'),
+  );
+getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
+
+loadSettings($aCMDResult['project-dir'] ?? getcwd());
+
+@define('CONST_Database_DSN', getSetting('DATABASE_DSN'));
+@define('CONST_Default_Language', getSetting('DEFAULT_LANGUAGE', false));
+@define('CONST_Log_DB', getSettingBool('LOG_DB'));
+@define('CONST_Log_File', getSetting('LOG_FILE', false));
+@define('CONST_Max_Word_Frequency', getSetting('MAX_WORD_FREQUENCY'));
+@define('CONST_NoAccessControl', getSettingBool('CORS_NOACCESSCONTROL'));
+@define('CONST_Places_Max_ID_count', getSetting('LOOKUP_MAX_COUNT'));
+@define('CONST_PolygonOutput_MaximumTypes', getSetting('POLYGON_OUTPUT_MAX_TYPES'));
+@define('CONST_Search_BatchMode', getSettingBool('SEARCH_BATCH_MODE'));
+@define('CONST_Search_NameOnlySearchFrequencyThreshold', getSetting('SEARCH_NAME_ONLY_THRESHOLD'));
+@define('CONST_Term_Normalization_Rules', getSetting('TERM_NORMALIZATION'));
+@define('CONST_Use_Aux_Location_data', getSettingBool('USE_AUX_LOCATION_DATA'));
+@define('CONST_Use_US_Tiger_Data', getSettingBool('USE_US_TIGER_DATA'));
+@define('CONST_MapIcon_URL', getSetting('MAPICON_URL', false));
+
+
+$oDB = new Nominatim\DB;
+$oDB->connect();
+
+if (isset($aCMDResult['nodedupe'])) $aCMDResult['dedupe'] = 'false';
+
+$oParams = new Nominatim\ParameterParser($aCMDResult);
+
+$aSearchParams = array(
+                     'search',
+                     'amenity',
+                     'street',
+                     'city',
+                     'county',
+                     'state',
+                     'country',
+                     'postalcode'
+                 );
+
+if (!$oParams->hasSetAny($aSearchParams)) {
+    showUsage($aCMDOptions, true);
+    return 1;
+}
+
+$oGeocode = new Nominatim\Geocode($oDB);
+
+$oGeocode->setLanguagePreference($oParams->getPreferredLanguages(false));
+$oGeocode->setReverseInPlan(true);
+$oGeocode->loadParamArray($oParams);
+
+if ($oParams->getBool('search')) {
+    $oGeocode->setQuery($aCMDResult['search']);
+} else {
+    $oGeocode->setQueryFromParams($oParams);
+}
+
+$aSearchResults = $oGeocode->lookup();
+
+echo json_encode($aSearchResults, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)."\n";

lib-php/admin/setup.php

@@ -0,0 +1,218 @@
+<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
+
+require_once(CONST_LibDir.'/init-cmd.php');
+require_once(CONST_LibDir.'/setup/SetupClass.php');
+require_once(CONST_LibDir.'/setup_functions.php');
+ini_set('memory_limit', '800M');
+
+use Nominatim\Setup\SetupFunctions as SetupFunctions;
+
+// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
+$aCMDOptions
+= array(
+   'Create and setup nominatim search system',
+   array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+   array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+   array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+
+   array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
+   array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
+
+   array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
+
+   array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
+   array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
+   array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
+   array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
+   array('reverse-only', '', 0, 1, 0, 0, 'bool', 'Do not create search tables and indexes'),
+   array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
+   array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
+   array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
+   array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
+   array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
+   array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
+   array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
+   array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
+   array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
+   array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
+   array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
+   array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
+   array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
+   array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
+   array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
+   array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
+   array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'),
+   array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'),
+   array('setup-website', '', 0, 1, 0, 0, 'bool', 'Used to compile environment variables for the website'),
+   array('project-dir', '', 0, 1, 1, 1, 'realpath', 'Base directory of the Nominatim installation (default: .)'),
+  );
+
+// $aCMDOptions passed to getCmdOpt by reference
+getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
+
+loadSettings($aCMDResult['project-dir'] ?? getcwd());
+setupHTTPProxy();
+
+$bDidSomething = false;
+
+$oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
+
+// by default, use all but one processor, but never more than 15.
+$iInstances = max(1, $aCMDResult['threads'] ?? (min(16, getProcessorCount()) - 1));
+
+function run($oCmd)
+{
+    global $iInstances;
+    global $aCMDResult;
+    $oCmd->addParams('--threads', $iInstances);
+    if ($aCMDResult['ignore-errors'] ?? false) {
+        $oCmd->addParams('--ignore-errors');
+    }
+    if ($aCMDResult['quiet'] ?? false) {
+        $oCmd->addParams('--quiet');
+    }
+    if ($aCMDResult['verbose'] ?? false) {
+        $oCmd->addParams('--verbose');
+    }
+    $oCmd->run(true);
+}
+
+
+//*******************************************************
+// Making some sanity check:
+// Check if osm-file is set and points to a valid file
+if ($aCMDResult['import-data'] || $aCMDResult['all']) {
+    // to remain in /lib/setup_functions.php function
+    checkInFile($aCMDResult['osm-file']);
+}
+
+// ******************************************************
+// instantiate Setup class
+$oSetup = new SetupFunctions($aCMDResult);
+
+// *******************************************************
+// go through complete process if 'all' is selected or start selected functions
+if ($aCMDResult['create-db'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    run((clone($oNominatimCmd))->addParams('transition', '--create-db'));
+}
+
+if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    $oCmd = (clone($oNominatimCmd))->addParams('transition', '--setup-db');
+
+    if ($aCMDResult['no-partitions'] ?? false) {
+        $oCmd->addParams('--no-partitions');
+    }
+
+    run($oCmd);
+}
+
+if ($aCMDResult['import-data'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    $oCmd = (clone($oNominatimCmd))
+        ->addParams('transition', '--import-data')
+        ->addParams('--osm-file', $aCMDResult['osm-file']);
+    if ($aCMDResult['drop'] ?? false) {
+        $oCmd->addParams('--drop');
+    }
+
+    run($oCmd);
+}
+
+if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    $oSetup->createSqlFunctions();
+}
+
+if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-tables');
+
+    if ($aCMDResult['reverse-only'] ?? false) {
+        $oCmd->addParams('--reverse-only');
+    }
+
+    run($oCmd);
+}
+
+if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    run((clone($oNominatimCmd))->addParams('transition', '--create-partition-tables'));
+}
+
+if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    $oSetup->createSqlFunctions(); // also create partition functions
+}
+
+if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    // ignore errors!
+    (clone($oNominatimCmd))->addParams('refresh', '--wiki-data')->run();
+}
+
+if ($aCMDResult['load-data'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    run((clone($oNominatimCmd))->addParams('transition', '--load-data'));
+}
+
+if ($aCMDResult['import-tiger-data']) {
+    $bDidSomething = true;
+    $sTigerPath = getSetting('TIGER_DATA_PATH', CONST_InstallDir.'/tiger');
+    run((clone($oNominatimCmd))->addParams('transition', '--tiger-data', $sTigerPath));
+}
+
+if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    $oSetup->calculatePostcodes($aCMDResult['all']);
+}
+
+if ($aCMDResult['index'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    $oCmd = (clone($oNominatimCmd))->addParams('transition', '--index');
+    if ($aCMDResult['index-noanalyse'] ?? false) {
+        $oCmd->addParams('--no-analyse');
+    }
+
+    run($oCmd);
+}
+
+if ($aCMDResult['drop']) {
+    $bDidSomething = true;
+    run((clone($oNominatimCmd))->addParams('freeze'));
+}
+
+if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+
+    $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-search-indices');
+
+    if ($aCMDResult['drop'] ?? false) {
+        $oCmd->addParams('--drop');
+    }
+
+    run($oCmd);
+}
+
+if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    run(clone($oNominatimCmd))->addParams('transition', '--create-country-names');
+}
+
+if ($aCMDResult['setup-website'] || $aCMDResult['all']) {
+    $bDidSomething = true;
+    run((clone($oNominatimCmd))->addParams('refresh', '--website'));
+}
+
+// ******************************************************
+// If we did something, repeat the warnings
+if (!$bDidSomething) {
+    showUsage($aCMDOptions, true);
+} else {
+    echo "Summary of warnings:\n\n";
+    repeatWarnings();
+    echo "\n";
+    info('Setup finished.');
+}

lib-php/admin/specialphrases.php

@@ -0,0 +1,11 @@
+
+<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
+
+require_once(CONST_LibDir.'/init-cmd.php');
+
+loadSettings(getcwd());
+
+(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
+    ->addParams('special-phrases', '--import-from-wiki')
+    ->run();

lib-php/admin/update.php

@@ -0,0 +1,236 @@
+<?php
+@define('CONST_LibDir', dirname(dirname(__FILE__)));
+
+require_once(CONST_LibDir.'/init-cmd.php');
+require_once(CONST_LibDir.'/setup_functions.php');
+require_once(CONST_LibDir.'/setup/SetupClass.php');
+
+ini_set('memory_limit', '800M');
+
+use Nominatim\Setup\SetupFunctions as SetupFunctions;
+
+// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
+$aCMDOptions
+= array(
+   'Import / update / index osm data',
+   array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+   array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+   array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+
+   array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
+   array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
+   array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
+   array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
+   array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
+   array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
+
+   array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
+
+   array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
+   array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
+   array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
+
+   array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
+   array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
+   array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
+   array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
+
+   array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
+   array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
+   array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
+
+   array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
+   array('update-address-levels', '', 0, 1, 0, 0, 'bool', 'Reimport address level configuration (EXPERT)'),
+   array('recompute-importance', '', 0, 1, 0, 0, 'bool', 'Recompute place importances'),
+
+   array('project-dir', '', 0, 1, 1, 1, 'realpath', 'Base directory of the Nominatim installation (default: .)'),
+  );
+
+getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
+
+loadSettings($aCMDResult['project-dir'] ?? getcwd());
+setupHTTPProxy();
+
+if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
+if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
+
+date_default_timezone_set('Etc/UTC');
+
+$oDB = new Nominatim\DB();
+$oDB->connect();
+$fPostgresVersion = $oDB->getPostgresVersion();
+
+$aDSNInfo = Nominatim\DB::parseDSN(getSetting('DATABASE_DSN'));
+if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
+
+// cache memory to be used by osm2pgsql, should not be more than the available memory
+$iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
+if ($iCacheMemory + 500 > getTotalMemoryMB()) {
+    $iCacheMemory = getCacheMemoryMB();
+    echo "WARNING: resetting cache memory to $iCacheMemory\n";
+}
+
+$oOsm2pgsqlCmd = (new \Nominatim\Shell(getOsm2pgsqlBinary()))
+                 ->addParams('--hstore')
+                 ->addParams('--latlong')
+                 ->addParams('--append')
+                 ->addParams('--slim')
+                 ->addParams('--with-forward-dependencies', 'false')
+                 ->addParams('--log-progress', 'true')
+                 ->addParams('--number-processes', 1)
+                 ->addParams('--cache', $iCacheMemory)
+                 ->addParams('--output', 'gazetteer')
+                 ->addParams('--style', getImportStyle())
+                 ->addParams('--database', $aDSNInfo['database'])
+                 ->addParams('--port', $aDSNInfo['port']);
+
+if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
+    $oOsm2pgsqlCmd->addParams('--host', $aDSNInfo['hostspec']);
+}
+if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
+    $oOsm2pgsqlCmd->addParams('--user', $aDSNInfo['username']);
+}
+if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
+    $oOsm2pgsqlCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']);
+}
+if (getSetting('FLATNODE_FILE')) {
+    $oOsm2pgsqlCmd->addParams('--flat-nodes', getSetting('FLATNODE_FILE'));
+}
+if ($fPostgresVersion >= 11.0) {
+    $oOsm2pgsqlCmd->addEnvPair(
+        'PGOPTIONS',
+        '-c jit=off -c max_parallel_workers_per_gather=0'
+    );
+}
+
+$oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
+
+function run($oCmd)
+{
+    global $aCMDResult;
+    if ($aCMDResult['quiet'] ?? false) {
+        $oCmd->addParams('--quiet');
+    }
+    if ($aCMDResult['verbose'] ?? false) {
+        $oCmd->addParams('--verbose');
+    }
+    $oCmd->run(true);
+}
+
+
+if ($aResult['init-updates']) {
+    $oCmd = (clone($oNominatimCmd))->addParams('replication', '--init');
+
+    if ($aResult['no-update-functions']) {
+        $oCmd->addParams('--no-update-functions');
+    }
+
+    run($oCmd);
+}
+
+if ($aResult['check-for-updates']) {
+    exit((clone($oNominatimCmd))->addParams('replication', '--check-for-updates')->run());
+}
+
+if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
+    // import diffs and files directly (e.g. from osmosis --rri)
+    $sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];
+
+    if (!file_exists($sNextFile)) {
+        fail("Cannot open $sNextFile\n");
+    }
+
+    // Import the file
+    $oCMD = (clone $oOsm2pgsqlCmd)->addParams($sNextFile);
+    echo $oCMD->escapedCmd()."\n";
+    $iRet = $oCMD->run();
+
+    if ($iRet) {
+        fail("Error from osm2pgsql, $iRet\n");
+    }
+
+    // Don't update the import status - we don't know what this file contains
+}
+
+if ($aResult['calculate-postcodes']) {
+    run((clone($oNominatimCmd))->addParams('refresh', '--postcodes'));
+}
+
+$sTemporaryFile = CONST_InstallDir.'/osmosischange.osc';
+$bHaveDiff = false;
+$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
+$sContentURL = '';
+if (isset($aResult['import-node']) && $aResult['import-node']) {
+    if ($bUseOSMApi) {
+        $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
+    } else {
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
+    }
+}
+
+if (isset($aResult['import-way']) && $aResult['import-way']) {
+    if ($bUseOSMApi) {
+        $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
+    } else {
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');%3E;);out%20meta;';
+    }
+}
+
+if (isset($aResult['import-relation']) && $aResult['import-relation']) {
+    if ($bUseOSMApi) {
+        $sContentURL = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
+    } else {
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=(rel(id:'.$aResult['import-relation'].');%3E;);out%20meta;';
+    }
+}
+
+if ($sContentURL) {
+    file_put_contents($sTemporaryFile, file_get_contents($sContentURL));
+    $bHaveDiff = true;
+}
+
+if ($bHaveDiff) {
+    // import generated change file
+
+    $oCMD = (clone $oOsm2pgsqlCmd)->addParams($sTemporaryFile);
+    echo $oCMD->escapedCmd()."\n";
+
+    $iRet = $oCMD->run();
+    if ($iRet) {
+        fail("osm2pgsql exited with error level $iRet\n");
+    }
+}
+
+if ($aResult['recompute-word-counts']) {
+    run((clone($oNominatimCmd))->addParams('refresh', '--word-counts'));
+}
+
+if ($aResult['index']) {
+    run((clone $oNominatimCmd)
+        ->addParams('index', '--minrank', $aResult['index-rank'])
+        ->addParams('--threads', $aResult['index-instances']));
+}
+
+if ($aResult['update-address-levels']) {
+    run((clone($oNominatimCmd))->addParams('refresh', '--address-levels'));
+}
+
+if ($aResult['recompute-importance']) {
+    run((clone($oNominatimCmd))->addParams('refresh', '--importance'));
+}
+
+if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
+    $oCmd = (clone($oNominatimCmd))
+              ->addParams('replication')
+              ->addParams('--threads', $aResult['index-instances']);
+
+    if (!$aResult['import-osmosis-all']) {
+        $oCmd->addParams('--once');
+    }
+
+    if ($aResult['no-index']) {
+        $oCmd->addParams('--no-index');
+    }
+
+    run($oCmd);
+}

lib-php/admin/warm.php

@@ -3,6 +3,7 @@
 
 require_once(CONST_LibDir.'/init-cmd.php');
 require_once(CONST_LibDir.'/log.php');
+require_once(CONST_LibDir.'/Geocode.php');
 require_once(CONST_LibDir.'/PlaceLookup.php');
 require_once(CONST_LibDir.'/ReverseGeocode.php');
 
@@ -25,16 +26,17 @@ loadSettings($aCMDResult['project-dir'] ?? getcwd());
 @define('CONST_Default_Language', getSetting('DEFAULT_LANGUAGE', false));
 @define('CONST_Log_DB', getSettingBool('LOG_DB'));
 @define('CONST_Log_File', getSetting('LOG_FILE', false));
+@define('CONST_Max_Word_Frequency', getSetting('MAX_WORD_FREQUENCY'));
 @define('CONST_NoAccessControl', getSettingBool('CORS_NOACCESSCONTROL'));
 @define('CONST_Places_Max_ID_count', getSetting('LOOKUP_MAX_COUNT'));
 @define('CONST_PolygonOutput_MaximumTypes', getSetting('POLYGON_OUTPUT_MAX_TYPES'));
 @define('CONST_Search_BatchMode', getSettingBool('SEARCH_BATCH_MODE'));
 @define('CONST_Search_NameOnlySearchFrequencyThreshold', getSetting('SEARCH_NAME_ONLY_THRESHOLD'));
+@define('CONST_Term_Normalization_Rules', getSetting('TERM_NORMALIZATION'));
+@define('CONST_Use_Aux_Location_data', getSettingBool('USE_AUX_LOCATION_DATA'));
 @define('CONST_Use_US_Tiger_Data', getSettingBool('USE_US_TIGER_DATA'));
 @define('CONST_MapIcon_URL', getSetting('MAPICON_URL', false));
-@define('CONST_TokenizerDir', CONST_InstallDir.'/tokenizer');
 
-require_once(CONST_LibDir.'/Geocode.php');
 
 $oDB = new Nominatim\DB();
 $oDB->connect();
@@ -62,15 +64,11 @@ if (!$aResult['search-only']) {
     $oPlaceLookup->setLanguagePreference(array('en'));
 
     echo 'Warm reverse: ';
-    if ($bVerbose) {
-        echo "\n";
-    }
+    if ($bVerbose) echo "\n";
     for ($i = 0; $i < 1000; $i++) {
         $fLat = rand(-9000, 9000) / 100;
         $fLon = rand(-18000, 18000) / 100;
-        if ($bVerbose) {
-            echo "$fLat, $fLon = ";
-        }
+        if ($bVerbose) echo "$fLat, $fLon = ";
 
         $oLookup = $oReverseGeocode->lookup($fLat, $fLon);
         $aSearchResults = $oLookup ? $oPlaceLookup->lookup(array($oLookup->iId => $oLookup)) : null;
@@ -83,14 +81,10 @@ if (!$aResult['reverse-only']) {
     $oGeocode = new Nominatim\Geocode($oDB);
 
     echo 'Warm search: ';
-    if ($bVerbose) {
-        echo "\n";
-    }
+    if ($bVerbose) echo "\n";
     $sSQL = 'SELECT word FROM word WHERE word is not null ORDER BY search_name_count DESC LIMIT 1000';
     foreach ($oDB->getCol($sSQL) as $sWord) {
-        if ($bVerbose) {
-            echo "$sWord = ";
-        }
+        if ($bVerbose) echo "$sWord = ";
 
         $oGeocode->setLanguagePreference(array('en'));
         $oGeocode->setQuery($sWord);

lib-php/cmd.php

@@ -9,12 +9,8 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
 
     foreach ($aSpec as $aLine) {
         if (is_array($aLine)) {
-            if ($aLine[0]) {
-                $aQuick['--'.$aLine[0]] = $aLine;
-            }
-            if ($aLine[1]) {
-                $aQuick['-'.$aLine[1]] = $aLine;
-            }
+            if ($aLine[0]) $aQuick['--'.$aLine[0]] = $aLine;
+            if ($aLine[1]) $aQuick['-'.$aLine[1]] = $aLine;
             $aCounts[$aLine[0]] = 0;
         }
     }
@@ -32,9 +28,7 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
                     $xVal = array();
                     for ($n = $aLine[4]; $i < $iSize && $n; $n--) {
                         $i++;
-                        if ($i >= $iSize || $aArg[$i][0] == '-') {
-                            showUsage($aSpec, $bExitOnError, 'Parameter of  \''.$aLine[0].'\' is missing');
-                        }
+                        if ($i >= $iSize || $aArg[$i][0] == '-') showUsage($aSpec, $bExitOnError, 'Parameter of  \''.$aLine[0].'\' is missing');
 
                         switch ($aLine[6]) {
                             case 'realpath':
@@ -62,9 +56,7 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
                                 break;
                         }
                     }
-                    if ($aLine[4] == 1) {
-                        $xVal = $xVal[0];
-                    }
+                    if ($aLine[4] == 1) $xVal = $xVal[0];
                 } else {
                     $xVal = true;
                 }
@@ -73,9 +65,7 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
             }
 
             if ($aLine[3] > 1) {
-                if (!array_key_exists($aLine[0], $aResult)) {
-                    $aResult[$aLine[0]] = array();
-                }
+                if (!array_key_exists($aLine[0], $aResult)) $aResult[$aLine[0]] = array();
                 $aResult[$aLine[0]][] = $xVal;
             } else {
                 $aResult[$aLine[0]] = $xVal;
@@ -85,23 +75,18 @@ function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnkn
         }
     }
 
-    if (array_key_exists('help', $aResult)) {
-        showUsage($aSpec);
-    }
-    if ($bUnknown && $bExitOnUnknown) {
-        showUsage($aSpec, $bExitOnError, 'Unknown option \''.$bUnknown.'\'');
-    }
+    if (array_key_exists('help', $aResult)) showUsage($aSpec);
+    if ($bUnknown && $bExitOnUnknown) showUsage($aSpec, $bExitOnError, 'Unknown option \''.$bUnknown.'\'');
 
     foreach ($aSpec as $aLine) {
         if (is_array($aLine)) {
-            if ($aCounts[$aLine[0]] < $aLine[2]) {
-                showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is missing');
-            }
-            if ($aCounts[$aLine[0]] > $aLine[3]) {
-                showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is pressent too many times');
-            }
-            if ($aLine[6] == 'bool' && !array_key_exists($aLine[0], $aResult)) {
-                $aResult[$aLine[0]] = false;
+            if ($aCounts[$aLine[0]] < $aLine[2]) showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is missing');
+            if ($aCounts[$aLine[0]] > $aLine[3]) showUsage($aSpec, $bExitOnError, 'Option \''.$aLine[0].'\' is pressent too many times');
+            switch ($aLine[6]) {
+                case 'bool':
+                    if (!array_key_exists($aLine[0], $aResult))
+                        $aResult[$aLine[0]] = false;
+                    break;
             }
         }
     }
@@ -124,12 +109,8 @@ function showUsage($aSpec, $bExit = false, $sError = false)
                 echo "\n";
             }
             $aNames = array();
-            if ($aLine[1]) {
-                $aNames[] = '-'.$aLine[1];
-            }
-            if ($aLine[0]) {
-                $aNames[] = '--'.$aLine[0];
-            }
+            if ($aLine[1]) $aNames[] = '-'.$aLine[1];
+            if ($aLine[0]) $aNames[] = '--'.$aLine[0];
             $sName = join(', ', $aNames);
             echo '  '.$sName.str_repeat(' ', 30-strlen($sName)).$aLine[7]."\n";
         } else {
@@ -163,6 +144,58 @@ function repeatWarnings()
 }
 
 
+function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreErrors = false)
+{
+    // Convert database DSN to psql parameters
+    $aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN'));
+    if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
+
+    $oCmd = new \Nominatim\Shell('psql');
+    $oCmd->addParams('--port', $aDSNInfo['port']);
+    $oCmd->addParams('--dbname', $aDSNInfo['database']);
+    if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
+        $oCmd->addParams('--host', $aDSNInfo['hostspec']);
+    }
+    if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
+        $oCmd->addParams('--username', $aDSNInfo['username']);
+    }
+    if (isset($aDSNInfo['password'])) {
+        $oCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']);
+    }
+    if (!$bVerbose) {
+        $oCmd->addParams('--quiet');
+    }
+    if ($bfatal && !$bIgnoreErrors) {
+        $oCmd->addParams('-v', 'ON_ERROR_STOP=1');
+    }
+
+    $aDescriptors = array(
+                     0 => array('pipe', 'r'),
+                     1 => STDOUT,
+                     2 => STDERR
+                    );
+    $ahPipes = null;
+    $hProcess = @proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
+    if (!is_resource($hProcess)) {
+        fail('unable to start pgsql');
+    }
+
+    if (!$bVerbose) {
+        fwrite($ahPipes[0], 'set client_min_messages to WARNING;');
+    }
+
+    while (strlen($sScript)) {
+        $iWritten = fwrite($ahPipes[0], $sScript);
+        if ($iWritten <= 0) break;
+        $sScript = substr($sScript, $iWritten);
+    }
+    fclose($ahPipes[0]);
+    $iReturn = proc_close($hProcess);
+    if ($bfatal && $iReturn > 0) {
+        fail("pgsql returned with error code ($iReturn)");
+    }
+}
+
 function setupHTTPProxy()
 {
     if (!getSettingBool('HTTP_PROXY')) {

lib-php/init-website.php

@@ -12,7 +12,7 @@ require_once(CONST_Debug ? 'DebugHtml.php' : 'DebugNone.php');
 
 function userError($sMsg)
 {
-    throw new \Exception($sMsg, 400);
+    throw new Exception($sMsg, 400);
 }
 
 
@@ -37,7 +37,7 @@ function shutdown_exception_handler_xml()
 {
     $error = error_get_last();
     if ($error !== null && $error['type'] === E_ERROR) {
-        exception_handler_xml(new \Exception($error['message'], 500));
+        exception_handler_xml(new Exception($error['message'], 500));
     }
 }
 
@@ -45,7 +45,7 @@ function shutdown_exception_handler_json()
 {
     $error = error_get_last();
     if ($error !== null && $error['type'] === E_ERROR) {
-        exception_handler_json(new \Exception($error['message'], 500));
+        exception_handler_json(new Exception($error['message'], 500));
     }
 }
 
@@ -81,10 +81,6 @@ if (CONST_NoAccessControl) {
         header('Access-Control-Allow-Headers: '.$_SERVER['HTTP_ACCESS_CONTROL_REQUEST_HEADERS']);
     }
 }
-if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] == 'OPTIONS') {
-    exit;
-}
+if (isset($_SERVER['REQUEST_METHOD']) && $_SERVER['REQUEST_METHOD'] == 'OPTIONS') exit;
 
-if (CONST_Debug) {
-    header('Content-type: text/html; charset=utf-8');
-}
+if (CONST_Debug) header('Content-type: text/html; charset=utf-8');

lib-php/lib.php

@@ -6,7 +6,10 @@ function loadSettings($sProjectDir)
     // Temporary hack to set the direcory via environment instead of
     // the installed scripts. Neither setting is part of the official
     // set of settings.
+    defined('CONST_DataDir') or define('CONST_DataDir', $_SERVER['NOMINATIM_DATADIR']);
+    defined('CONST_SqlDir') or define('CONST_SqlDir', $_SERVER['NOMINATIM_SQLDIR']);
     defined('CONST_ConfigDir') or define('CONST_ConfigDir', $_SERVER['NOMINATIM_CONFIGDIR']);
+    defined('CONST_Default_ModulePath') or define('CONST_Default_ModulePath', $_SERVER['NOMINATIM_DATABASE_MODULE_SRC_PATH']);
 }
 
 function getSetting($sConfName, $sDefault = null)
@@ -29,14 +32,22 @@ function getSettingBool($sConfName)
            || strcmp($sVal, '1') == 0;
 }
 
+function getSettingConfig($sConfName, $sSystemConfig)
+{
+    $sValue = $_SERVER['NOMINATIM_'.$sConfName];
+
+    if (!$sValue) {
+        return CONST_ConfigDir.'/'.$sSystemConfig;
+    }
+
+    return $sValue;
+}
+
 function fail($sError, $sUserError = false)
 {
-    if (!$sUserError) {
-        $sUserError = $sError;
-    }
+    if (!$sUserError) $sUserError = $sError;
     error_log('ERROR: '.$sError);
-    var_dump($sUserError);
-    echo "\n";
+    var_dump($sUserError)."\n";
     exit(-1);
 }
 
@@ -84,9 +95,8 @@ function getDatabaseDate(&$oDB)
 
 function byImportance($a, $b)
 {
-    if ($a['importance'] != $b['importance']) {
+    if ($a['importance'] != $b['importance'])
         return ($a['importance'] > $b['importance']?-1:1);
-    }
 
     return $a['foundorder'] <=> $b['foundorder'];
 }
@@ -221,8 +231,6 @@ function closestHouseNumber($aRow)
 if (!function_exists('array_key_last')) {
     function array_key_last(array $array)
     {
-        if (!empty($array)) {
-            return key(array_slice($array, -1, 1, true));
-        }
+        if (!empty($array)) return key(array_slice($array, -1, 1, true));
     }
 }

lib-php/log.php

@@ -5,23 +5,15 @@ function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array())
 {
     $fStartTime = microtime(true);
     $aStartTime = explode('.', $fStartTime);
-    if (!isset($aStartTime[1])) {
-        $aStartTime[1] = '0';
-    }
+    if (!isset($aStartTime[1])) $aStartTime[1] = '0';
 
     $sOutputFormat = '';
-    if (isset($_GET['format'])) {
-        $sOutputFormat = $_GET['format'];
-    }
+    if (isset($_GET['format'])) $sOutputFormat = $_GET['format'];
 
     if ($sType == 'reverse') {
         $sOutQuery = (isset($_GET['lat'])?$_GET['lat']:'').'/';
-        if (isset($_GET['lon'])) {
-            $sOutQuery .= $_GET['lon'];
-        }
-        if (isset($_GET['zoom'])) {
-            $sOutQuery .= '/'.$_GET['zoom'];
-        }
+        if (isset($_GET['lon'])) $sOutQuery .= $_GET['lon'];
+        if (isset($_GET['zoom'])) $sOutQuery .= '/'.$_GET['zoom'];
     } else {
         $sOutQuery = $sQuery;
     }
@@ -36,15 +28,13 @@ function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array())
             );
 
     if (CONST_Log_DB) {
-        if (isset($_GET['email'])) {
+        if (isset($_GET['email']))
             $sUserAgent = $_GET['email'];
-        } elseif (isset($_SERVER['HTTP_REFERER'])) {
+        elseif (isset($_SERVER['HTTP_REFERER']))
             $sUserAgent = $_SERVER['HTTP_REFERER'];
-        } elseif (isset($_SERVER['HTTP_USER_AGENT'])) {
+        elseif (isset($_SERVER['HTTP_USER_AGENT']))
             $sUserAgent = $_SERVER['HTTP_USER_AGENT'];
-        } else {
-            $sUserAgent = '';
-        }
+        else $sUserAgent = '';
         $sSQL = 'insert into new_query_log (type,starttime,query,ipaddress,useragent,language,format,searchterm)';
         $sSQL .= ' values (';
         $sSQL .= join(',', $oDB->getDBQuotedList(array(
@@ -70,9 +60,7 @@ function logEnd(&$oDB, $hLog, $iNumResults)
 
     if (CONST_Log_DB) {
         $aEndTime = explode('.', $fEndTime);
-        if (!$aEndTime[1]) {
-            $aEndTime[1] = '0';
-        }
+        if (!$aEndTime[1]) $aEndTime[1] = '0';
         $sEndTime = date('Y-m-d H:i:s', $aEndTime[0]).'.'.$aEndTime[1];
 
         $sSQL = 'update new_query_log set endtime = '.$oDB->getDBQuoted($sEndTime).', results = '.$iNumResults;

lib-php/migration/PhraseSettingsToJson.php

@@ -8,12 +8,10 @@ if (file_exists($phpPhraseSettingsFile) && !file_exists($jsonPhraseSettingsFile)
 
     $data = array();
 
-    if (isset($aTagsBlacklist)) {
+    if (isset($aTagsBlacklist))
         $data['blackList'] = $aTagsBlacklist;
-    }
-    if (isset($aTagsWhitelist)) {
+    if (isset($aTagsWhitelist))
         $data['whiteList'] = $aTagsWhitelist;
-    }
 
     $jsonFile = fopen($jsonPhraseSettingsFile, 'w');
     fwrite($jsonFile, json_encode($data));

lib-php/output.php

@@ -3,26 +3,14 @@
 
 function formatOSMType($sType, $bIncludeExternal = true)
 {
-    if ($sType == 'N') {
-        return 'node';
-    }
-    if ($sType == 'W') {
-        return 'way';
-    }
-    if ($sType == 'R') {
-        return 'relation';
-    }
+    if ($sType == 'N') return 'node';
+    if ($sType == 'W') return 'way';
+    if ($sType == 'R') return 'relation';
 
-    if (!$bIncludeExternal) {
-        return '';
-    }
+    if (!$bIncludeExternal) return '';
 
-    if ($sType == 'T') {
-        return 'way';
-    }
-    if ($sType == 'I') {
-        return 'way';
-    }
+    if ($sType == 'T') return 'way';
+    if ($sType == 'I') return 'way';
 
     // not handled: P, L
 

lib-php/setup/SetupClass.php

@@ -0,0 +1,261 @@
+<?php
+
+namespace Nominatim\Setup;
+
+require_once(CONST_LibDir.'/Shell.php');
+
+class SetupFunctions
+{
+    protected $iInstances;
+    protected $aDSNInfo;
+    protected $bQuiet;
+    protected $bVerbose;
+    protected $sIgnoreErrors;
+    protected $bEnableDiffUpdates;
+    protected $bEnableDebugStatements;
+    protected $bDrop;
+    protected $oDB = null;
+    protected $oNominatimCmd;
+
+    public function __construct(array $aCMDResult)
+    {
+        // by default, use all but one processor, but never more than 15.
+        $this->iInstances = isset($aCMDResult['threads'])
+            ? $aCMDResult['threads']
+            : (min(16, getProcessorCount()) - 1);
+
+        if ($this->iInstances < 1) {
+            $this->iInstances = 1;
+            warn('resetting threads to '.$this->iInstances);
+        }
+
+        // parse database string
+        $this->aDSNInfo = \Nominatim\DB::parseDSN(getSetting('DATABASE_DSN'));
+        if (!isset($this->aDSNInfo['port'])) {
+            $this->aDSNInfo['port'] = 5432;
+        }
+
+        // setting member variables based on command line options stored in $aCMDResult
+        $this->bQuiet = isset($aCMDResult['quiet']) && $aCMDResult['quiet'];
+        $this->bVerbose = $aCMDResult['verbose'];
+
+        //setting default values which are not set by the update.php array
+        if (isset($aCMDResult['ignore-errors'])) {
+            $this->sIgnoreErrors = $aCMDResult['ignore-errors'];
+        } else {
+            $this->sIgnoreErrors = false;
+        }
+        if (isset($aCMDResult['enable-debug-statements'])) {
+            $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements'];
+        } else {
+            $this->bEnableDebugStatements = false;
+        }
+        if (isset($aCMDResult['enable-diff-updates'])) {
+            $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates'];
+        } else {
+            $this->bEnableDiffUpdates = false;
+        }
+
+        $this->bDrop = isset($aCMDResult['drop']) && $aCMDResult['drop'];
+
+        $this->oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
+        if ($this->bQuiet) {
+            $this->oNominatimCmd->addParams('--quiet');
+        }
+        if ($this->bVerbose) {
+            $this->oNominatimCmd->addParams('--verbose');
+        }
+    }
+
+    public function calculatePostcodes($bCMDResultAll)
+    {
+        info('Calculate Postcodes');
+        $this->pgsqlRunScriptFile(CONST_SqlDir.'/postcode_tables.sql');
+
+        $sPostcodeFilename = CONST_InstallDir.'/gb_postcode_data.sql.gz';
+        if (file_exists($sPostcodeFilename)) {
+            $this->pgsqlRunScriptFile($sPostcodeFilename);
+        } else {
+            warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
+        }
+
+        $sPostcodeFilename = CONST_InstallDir.'/us_postcode_data.sql.gz';
+        if (file_exists($sPostcodeFilename)) {
+            $this->pgsqlRunScriptFile($sPostcodeFilename);
+        } else {
+            warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
+        }
+
+
+        $this->db()->exec('TRUNCATE location_postcode');
+
+        $sSQL  = 'INSERT INTO location_postcode';
+        $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
+        $sSQL .= "SELECT nextval('seq_place'), 1, country_code,";
+        $sSQL .= "       upper(trim (both ' ' from address->'postcode')) as pc,";
+        $sSQL .= '       ST_Centroid(ST_Collect(ST_Centroid(geometry)))';
+        $sSQL .= '  FROM placex';
+        $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
+        $sSQL .= '       AND geometry IS NOT null';
+        $sSQL .= ' GROUP BY country_code, pc';
+        $this->db()->exec($sSQL);
+
+        // only add postcodes that are not yet available in OSM
+        $sSQL  = 'INSERT INTO location_postcode';
+        $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
+        $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
+        $sSQL .= '       ST_SetSRID(ST_Point(x,y),4326)';
+        $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
+        $sSQL .= '        (SELECT postcode FROM location_postcode';
+        $sSQL .= "          WHERE country_code = 'us')";
+        $this->db()->exec($sSQL);
+
+        // add missing postcodes for GB (if available)
+        $sSQL  = 'INSERT INTO location_postcode';
+        $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
+        $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry";
+        $sSQL .= '  FROM gb_postcode WHERE postcode NOT IN';
+        $sSQL .= '           (SELECT postcode FROM location_postcode';
+        $sSQL .= "             WHERE country_code = 'gb')";
+        $this->db()->exec($sSQL);
+
+        if (!$bCMDResultAll) {
+            $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
+            $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
+            $this->db()->exec($sSQL);
+        }
+
+        $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
+        $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
+        $this->db()->exec($sSQL);
+    }
+
+    /**
+     * Return the connection to the database.
+     *
+     * @return Database object.
+     *
+     * Creates a new connection if none exists yet. Otherwise reuses the
+     * already established connection.
+     */
+    private function db()
+    {
+        if (is_null($this->oDB)) {
+            $this->oDB = new \Nominatim\DB();
+            $this->oDB->connect();
+        }
+
+        return $this->oDB;
+    }
+
+    private function pgsqlRunScript($sScript, $bfatal = true)
+    {
+        runSQLScript(
+            $sScript,
+            $bfatal,
+            $this->bVerbose,
+            $this->sIgnoreErrors
+        );
+    }
+
+    public function createSqlFunctions()
+    {
+        $oCmd = (clone($this->oNominatimCmd))
+                ->addParams('refresh', '--functions');
+
+        if (!$this->bEnableDiffUpdates) {
+            $oCmd->addParams('--no-diff-updates');
+        }
+
+        if ($this->bEnableDebugStatements) {
+            $oCmd->addParams('--enable-debug-statements');
+        }
+
+        $oCmd->run(!$this->sIgnoreErrors);
+    }
+
+    private function pgsqlRunScriptFile($sFilename)
+    {
+        if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
+
+        $oCmd = (new \Nominatim\Shell('psql'))
+                ->addParams('--port', $this->aDSNInfo['port'])
+                ->addParams('--dbname', $this->aDSNInfo['database']);
+
+        if (!$this->bVerbose) {
+            $oCmd->addParams('--quiet');
+        }
+        if (isset($this->aDSNInfo['hostspec'])) {
+            $oCmd->addParams('--host', $this->aDSNInfo['hostspec']);
+        }
+        if (isset($this->aDSNInfo['username'])) {
+            $oCmd->addParams('--username', $this->aDSNInfo['username']);
+        }
+        if (isset($this->aDSNInfo['password'])) {
+            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
+        }
+        $ahGzipPipes = null;
+        if (preg_match('/\\.gz$/', $sFilename)) {
+            $aDescriptors = array(
+                             0 => array('pipe', 'r'),
+                             1 => array('pipe', 'w'),
+                             2 => array('file', '/dev/null', 'a')
+                            );
+            $oZcatCmd = new \Nominatim\Shell('zcat', $sFilename);
+
+            $hGzipProcess = proc_open($oZcatCmd->escapedCmd(), $aDescriptors, $ahGzipPipes);
+            if (!is_resource($hGzipProcess)) fail('unable to start zcat');
+            $aReadPipe = $ahGzipPipes[1];
+            fclose($ahGzipPipes[0]);
+        } else {
+            $oCmd->addParams('--file', $sFilename);
+            $aReadPipe = array('pipe', 'r');
+        }
+        $aDescriptors = array(
+                         0 => $aReadPipe,
+                         1 => array('pipe', 'w'),
+                         2 => array('file', '/dev/null', 'a')
+                        );
+        $ahPipes = null;
+
+        $hProcess = proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
+        if (!is_resource($hProcess)) fail('unable to start pgsql');
+        // TODO: error checking
+        while (!feof($ahPipes[1])) {
+            echo fread($ahPipes[1], 4096);
+        }
+        fclose($ahPipes[1]);
+        $iReturn = proc_close($hProcess);
+        if ($iReturn > 0) {
+            fail("pgsql returned with error code ($iReturn)");
+        }
+        if ($ahGzipPipes) {
+            fclose($ahGzipPipes[1]);
+            proc_close($hGzipProcess);
+        }
+    }
+
+    private function replaceSqlPatterns($sSql)
+    {
+        $sSql = str_replace('{www-user}', getSetting('DATABASE_WEBUSER'), $sSql);
+
+        $aPatterns = array(
+                      '{ts:address-data}' => getSetting('TABLESPACE_ADDRESS_DATA'),
+                      '{ts:address-index}' => getSetting('TABLESPACE_ADDRESS_INDEX'),
+                      '{ts:search-data}' => getSetting('TABLESPACE_SEARCH_DATA'),
+                      '{ts:search-index}' =>  getSetting('TABLESPACE_SEARCH_INDEX'),
+                      '{ts:aux-data}' =>  getSetting('TABLESPACE_AUX_DATA'),
+                      '{ts:aux-index}' =>  getSetting('TABLESPACE_AUX_INDEX')
+        );
+
+        foreach ($aPatterns as $sPattern => $sTablespace) {
+            if ($sTablespace) {
+                $sSql = str_replace($sPattern, 'TABLESPACE "'.$sTablespace.'"', $sSql);
+            } else {
+                $sSql = str_replace($sPattern, '', $sSql);
+            }
+        }
+
+        return $sSql;
+    }
+}

lib-php/setup_functions.php

@@ -1,5 +1,20 @@
 <?php
 
+function checkInFile($sOSMFile)
+{
+    if (!isset($sOSMFile)) {
+        fail('missing --osm-file for data import');
+    }
+
+    if (!file_exists($sOSMFile)) {
+        fail('the path supplied to --osm-file does not exist');
+    }
+
+    if (!is_readable($sOSMFile)) {
+        fail('osm-file "' . $aCMDResult['osm-file'] . '" not readable');
+    }
+}
+
 function getOsm2pgsqlBinary()
 {
     $sBinary = getSetting('OSM2PGSQL_BINARY');

lib-php/template/address-geocodejson.php

@@ -5,11 +5,9 @@
 $aFilteredPlaces = array();
 
 if (empty($aPlace)) {
-    if (isset($sError)) {
+    if (isset($sError))
         $aFilteredPlaces['error'] = $sError;
-    } else {
-        $aFilteredPlaces['error'] = 'Unable to geocode';
-    }
+    else $aFilteredPlaces['error'] = 'Unable to geocode';
     javascript_renderData($aFilteredPlaces);
 } else {
     $aFilteredPlaces = array(
@@ -19,9 +17,7 @@ if (empty($aPlace)) {
                                         )
                        );
 
-    if (isset($aPlace['place_id'])) {
-        $aFilteredPlaces['properties']['geocoding']['place_id'] = $aPlace['place_id'];
-    }
+    if (isset($aPlace['place_id'])) $aFilteredPlaces['properties']['geocoding']['place_id'] = $aPlace['place_id'];
     $sOSMType = formatOSMType($aPlace['osm_type']);
     if ($sOSMType) {
         $aFilteredPlaces['properties']['geocoding']['osm_type'] = $sOSMType;

lib-php/template/address-geojson.php

@@ -3,11 +3,9 @@
 $aFilteredPlaces = array();
 
 if (empty($aPlace)) {
-    if (isset($sError)) {
+    if (isset($sError))
         $aFilteredPlaces['error'] = $sError;
-    } else {
-        $aFilteredPlaces['error'] = 'Unable to geocode';
-    }
+    else $aFilteredPlaces['error'] = 'Unable to geocode';
     javascript_renderData($aFilteredPlaces);
 } else {
     $aFilteredPlaces = array(
@@ -15,9 +13,7 @@ if (empty($aPlace)) {
                         'properties' => array()
                        );
 
-    if (isset($aPlace['place_id'])) {
-        $aFilteredPlaces['properties']['place_id'] = $aPlace['place_id'];
-    }
+    if (isset($aPlace['place_id'])) $aFilteredPlaces['properties']['place_id'] = $aPlace['place_id'];
     $sOSMType = formatOSMType($aPlace['osm_type']);
     if ($sOSMType) {
         $aFilteredPlaces['properties']['osm_type'] = $sOSMType;
@@ -40,12 +36,8 @@ if (empty($aPlace)) {
     if (isset($aPlace['address'])) {
         $aFilteredPlaces['properties']['address'] = $aPlace['address']->getAddressNames();
     }
-    if (isset($aPlace['sExtraTags'])) {
-        $aFilteredPlaces['properties']['extratags'] = $aPlace['sExtraTags'];
-    }
-    if (isset($aPlace['sNameDetails'])) {
-        $aFilteredPlaces['properties']['namedetails'] = $aPlace['sNameDetails'];
-    }
+    if (isset($aPlace['sExtraTags'])) $aFilteredPlaces['properties']['extratags'] = $aPlace['sExtraTags'];
+    if (isset($aPlace['sNameDetails'])) $aFilteredPlaces['properties']['namedetails'] = $aPlace['sNameDetails'];
 
     if (isset($aPlace['aBoundingBox'])) {
         $aFilteredPlaces['bbox'] = array(

lib-php/template/address-json.php

@@ -3,27 +3,19 @@
 $aFilteredPlaces = array();
 
 if (empty($aPlace)) {
-    if (isset($sError)) {
+    if (isset($sError))
         $aFilteredPlaces['error'] = $sError;
-    } else {
-        $aFilteredPlaces['error'] = 'Unable to geocode';
-    }
+    else $aFilteredPlaces['error'] = 'Unable to geocode';
 } else {
-    if (isset($aPlace['place_id'])) {
-        $aFilteredPlaces['place_id'] = $aPlace['place_id'];
-    }
+    if (isset($aPlace['place_id'])) $aFilteredPlaces['place_id'] = $aPlace['place_id'];
     $aFilteredPlaces['licence'] = 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright';
     $sOSMType = formatOSMType($aPlace['osm_type']);
     if ($sOSMType) {
         $aFilteredPlaces['osm_type'] = $sOSMType;
         $aFilteredPlaces['osm_id'] = $aPlace['osm_id'];
     }
-    if (isset($aPlace['lat'])) {
-        $aFilteredPlaces['lat'] = $aPlace['lat'];
-    }
-    if (isset($aPlace['lon'])) {
-        $aFilteredPlaces['lon'] = $aPlace['lon'];
-    }
+    if (isset($aPlace['lat'])) $aFilteredPlaces['lat'] = $aPlace['lat'];
+    if (isset($aPlace['lon'])) $aFilteredPlaces['lon'] = $aPlace['lon'];
 
     if ($sOutputFormat == 'jsonv2' || $sOutputFormat == 'geojson') {
         $aFilteredPlaces['place_rank'] = $aPlace['rank_search'];
@@ -43,12 +35,8 @@ if (empty($aPlace)) {
     if (isset($aPlace['address'])) {
         $aFilteredPlaces['address'] = $aPlace['address']->getAddressNames();
     }
-    if (isset($aPlace['sExtraTags'])) {
-        $aFilteredPlaces['extratags'] = $aPlace['sExtraTags'];
-    }
-    if (isset($aPlace['sNameDetails'])) {
-        $aFilteredPlaces['namedetails'] = $aPlace['sNameDetails'];
-    }
+    if (isset($aPlace['sExtraTags'])) $aFilteredPlaces['extratags'] = $aPlace['sExtraTags'];
+    if (isset($aPlace['sNameDetails'])) $aFilteredPlaces['namedetails'] = $aPlace['sNameDetails'];
 
     if (isset($aPlace['aBoundingBox'])) {
         $aFilteredPlaces['boundingbox'] = $aPlace['aBoundingBox'];

lib-php/template/address-xml.php

@@ -12,29 +12,17 @@ echo " querystring='".htmlspecialchars($_SERVER['QUERY_STRING'], ENT_QUOTES)."'"
 echo ">\n";
 
 if (empty($aPlace)) {
-    if (isset($sError)) {
+    if (isset($sError))
         echo "<error>$sError</error>";
-    } else {
-        echo '<error>Unable to geocode</error>';
-    }
+    else echo '<error>Unable to geocode</error>';
 } else {
     echo '<result';
-    if ($aPlace['place_id']) {
-        echo ' place_id="'.$aPlace['place_id'].'"';
-    }
+    if ($aPlace['place_id']) echo ' place_id="'.$aPlace['place_id'].'"';
     $sOSMType = formatOSMType($aPlace['osm_type']);
-    if ($sOSMType) {
-        echo ' osm_type="'.$sOSMType.'"'.' osm_id="'.$aPlace['osm_id'].'"';
-    }
-    if ($aPlace['ref']) {
-        echo ' ref="'.htmlspecialchars($aPlace['ref']).'"';
-    }
-    if (isset($aPlace['lat'])) {
-        echo ' lat="'.htmlspecialchars($aPlace['lat']).'"';
-    }
-    if (isset($aPlace['lon'])) {
-        echo ' lon="'.htmlspecialchars($aPlace['lon']).'"';
-    }
+    if ($sOSMType) echo ' osm_type="'.$sOSMType.'"'.' osm_id="'.$aPlace['osm_id'].'"';
+    if ($aPlace['ref']) echo ' ref="'.htmlspecialchars($aPlace['ref']).'"';
+    if (isset($aPlace['lat'])) echo ' lat="'.htmlspecialchars($aPlace['lat']).'"';
+    if (isset($aPlace['lon'])) echo ' lon="'.htmlspecialchars($aPlace['lon']).'"';
     if (isset($aPlace['aBoundingBox'])) {
         echo ' boundingbox="';
         echo join(',', $aPlace['aBoundingBox']);

lib-php/template/details-json.php

@@ -43,26 +43,29 @@ $aPlaceDetails['centroid'] = array(
 $aPlaceDetails['geometry'] = json_decode($aPointDetails['asgeojson']);
 
 $funcMapAddressLine = function ($aFull) {
-    return array(
-            'localname' => $aFull['localname'],
-            'place_id' => isset($aFull['place_id']) ? (int) $aFull['place_id'] : null,
-            'osm_id' => isset($aFull['osm_id']) ? (int) $aFull['osm_id'] : null,
-            'osm_type' => isset($aFull['osm_type']) ? $aFull['osm_type'] : null,
-            'place_type' => isset($aFull['place_type']) ? $aFull['place_type'] : null,
-            'class' => $aFull['class'],
-            'type' => $aFull['type'],
-            'admin_level' => isset($aFull['admin_level']) ? (int) $aFull['admin_level'] : null,
-            'rank_address' => $aFull['rank_address'] ? (int) $aFull['rank_address'] : null,
-            'distance' => (float) $aFull['distance'],
-            'isaddress' => isset($aFull['isaddress']) ? (bool) $aFull['isaddress'] : null
-           );
+    $aMapped = array(
+                'localname' => $aFull['localname'],
+                'place_id' => isset($aFull['place_id']) ? (int) $aFull['place_id'] : null,
+                'osm_id' => isset($aFull['osm_id']) ? (int) $aFull['osm_id'] : null,
+                'osm_type' => isset($aFull['osm_type']) ? $aFull['osm_type'] : null,
+                'place_type' => isset($aFull['place_type']) ? $aFull['place_type'] : null,
+                'class' => $aFull['class'],
+                'type' => $aFull['type'],
+                'admin_level' => isset($aFull['admin_level']) ? (int) $aFull['admin_level'] : null,
+                'rank_address' => $aFull['rank_address'] ? (int) $aFull['rank_address'] : null,
+                'distance' => (float) $aFull['distance'],
+                'isaddress' => isset($aFull['isaddress']) ? (bool) $aFull['isaddress'] : null
+               );
+
+    return $aMapped;
 };
 
 $funcMapKeyword = function ($aFull) {
-    return array(
-            'id' => (int) $aFull['word_id'],
-            'token' => $aFull['word_token']
-           );
+    $aMapped = array(
+                'id' => (int) $aFull['word_id'],
+                'token' => $aFull['word_token']
+               );
+    return $aMapped;
 };
 
 if ($aAddressLines) {
@@ -78,14 +81,10 @@ if ($bIncludeKeywords) {
 
     if ($aPlaceSearchNameKeywords) {
         $aPlaceDetails['keywords']['name'] = array_map($funcMapKeyword, $aPlaceSearchNameKeywords);
-    } else {
-        $aPlaceDetails['keywords']['name'] = array();
     }
 
     if ($aPlaceSearchAddressKeywords) {
         $aPlaceDetails['keywords']['address'] = array_map($funcMapKeyword, $aPlaceSearchAddressKeywords);
-    } else {
-        $aPlaceDetails['keywords']['address'] = array();
     }
 }
 
@@ -93,15 +92,11 @@ if ($bIncludeHierarchy) {
     if ($bGroupHierarchy) {
         $aPlaceDetails['hierarchy'] = array();
         foreach ($aHierarchyLines as $aAddressLine) {
-            if ($aAddressLine['type'] == 'yes') {
-                $sType = $aAddressLine['class'];
-            } else {
-                $sType = $aAddressLine['type'];
-            }
+            if ($aAddressLine['type'] == 'yes') $sType = $aAddressLine['class'];
+            else $sType = $aAddressLine['type'];
 
-            if (!isset($aPlaceDetails['hierarchy'][$sType])) {
+            if (!isset($aPlaceDetails['hierarchy'][$sType]))
                 $aPlaceDetails['hierarchy'][$sType] = array();
-            }
             $aPlaceDetails['hierarchy'][$sType][] = $funcMapAddressLine($aAddressLine);
         }
     } else {

lib-php/template/error-json.php

@@ -8,4 +8,4 @@
         $error['details'] = $exception->getFile() . '('. $exception->getLine() . ')';
     }
 
-    javascript_renderData(array('error' => $error));
+    echo javascript_renderData(array('error' => $error));

lib-php/template/search-batch-json.php

@@ -5,9 +5,7 @@ $aOutput['licence'] = 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm
 $aOutput['batch'] = array();
 
 foreach ($aBatchResults as $aSearchResults) {
-    if (!$aSearchResults) {
-        $aSearchResults = array();
-    }
+    if (!$aSearchResults) $aSearchResults = array();
     $aFilteredPlaces = array();
     foreach ($aSearchResults as $iResNum => $aPointDetails) {
         $aPlace = array(

lib-php/template/search-geocodejson.php

@@ -9,9 +9,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
                                )
               );
 
-    if (isset($aPointDetails['place_id'])) {
-        $aPlace['properties']['geocoding']['place_id'] = $aPointDetails['place_id'];
-    }
+    if (isset($aPointDetails['place_id'])) $aPlace['properties']['geocoding']['place_id'] = $aPointDetails['place_id'];
     $sOSMType = formatOSMType($aPointDetails['osm_type']);
     if ($sOSMType) {
         $aPlace['properties']['geocoding']['osm_type'] = $sOSMType;

lib-php/template/search-geojson.php

@@ -8,7 +8,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
                                 'place_id'=>$aPointDetails['place_id'],
                                )
               );
-
+    
     $sOSMType = formatOSMType($aPointDetails['osm_type']);
     if ($sOSMType) {
         $aPlace['properties']['osm_type'] = $sOSMType;
@@ -58,12 +58,8 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
     }
 
 
-    if (isset($aPointDetails['sExtraTags'])) {
-        $aPlace['properties']['extratags'] = $aPointDetails['sExtraTags'];
-    }
-    if (isset($aPointDetails['sNameDetails'])) {
-        $aPlace['properties']['namedetails'] = $aPointDetails['sNameDetails'];
-    }
+    if (isset($aPointDetails['sExtraTags'])) $aPlace['properties']['extratags'] = $aPointDetails['sExtraTags'];
+    if (isset($aPointDetails['sNameDetails'])) $aPlace['properties']['namedetails'] = $aPointDetails['sNameDetails'];
 
     $aFilteredPlaces[] = $aPlace;
 }

lib-php/template/search-json.php

@@ -6,7 +6,7 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
                'place_id'=>$aPointDetails['place_id'],
                'licence'=>'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright',
               );
-
+    
     $sOSMType = formatOSMType($aPointDetails['osm_type']);
     if ($sOSMType) {
         $aPlace['osm_type'] = $sOSMType;
@@ -60,12 +60,8 @@ foreach ($aSearchResults as $iResNum => $aPointDetails) {
         $aPlace['geokml'] = $aPointDetails['askml'];
     }
 
-    if (isset($aPointDetails['sExtraTags'])) {
-        $aPlace['extratags'] = $aPointDetails['sExtraTags'];
-    }
-    if (isset($aPointDetails['sNameDetails'])) {
-        $aPlace['namedetails'] = $aPointDetails['sNameDetails'];
-    }
+    if (isset($aPointDetails['sExtraTags'])) $aPlace['extratags'] = $aPointDetails['sExtraTags'];
+    if (isset($aPointDetails['sNameDetails'])) $aPlace['namedetails'] = $aPointDetails['sNameDetails'];
 
     $aFilteredPlaces[] = $aPlace;
 }

lib-php/template/search-xml.php

@@ -10,9 +10,7 @@ echo (isset($sXmlRootTag)?$sXmlRootTag:'searchresults');
 echo " timestamp='".date(DATE_RFC822)."'";
 echo " attribution='Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright'";
 echo " querystring='".htmlspecialchars($sQuery, ENT_QUOTES)."'";
-if (isset($aMoreParams['viewbox'])) {
-    echo " viewbox='".htmlspecialchars($aMoreParams['viewbox'], ENT_QUOTES)."'";
-}
+if (isset($aMoreParams['viewbox'])) echo " viewbox='".htmlspecialchars($aMoreParams['viewbox'], ENT_QUOTES)."'";
 if (isset($aMoreParams['exclude_place_ids'])) {
     echo " exclude_place_ids='".htmlspecialchars($aMoreParams['exclude_place_ids'])."'";
 }

lib-php/tokenizer/legacy_icu_tokenizer.php

@@ -1,246 +0,0 @@
-<?php
-
-namespace Nominatim;
-
-class Tokenizer
-{
-    private $oDB;
-
-    private $oNormalizer;
-    private $oTransliterator;
-    private $aCountryRestriction;
-
-    public function __construct(&$oDB)
-    {
-        $this->oDB =& $oDB;
-        $this->oNormalizer = \Transliterator::createFromRules(CONST_Term_Normalization_Rules);
-        $this->oTransliterator = \Transliterator::createFromRules(CONST_Transliteration);
-    }
-
-    public function checkStatus()
-    {
-        $sSQL = 'SELECT word_id FROM word limit 1';
-        $iWordID = $this->oDB->getOne($sSQL);
-        if ($iWordID === false) {
-            throw new \Exception('Query failed', 703);
-        }
-        if (!$iWordID) {
-            throw new \Exception('No value', 704);
-        }
-    }
-
-
-    public function setCountryRestriction($aCountries)
-    {
-        $this->aCountryRestriction = $aCountries;
-    }
-
-
-    public function normalizeString($sTerm)
-    {
-        if ($this->oNormalizer === null) {
-            return $sTerm;
-        }
-
-        return $this->oNormalizer->transliterate($sTerm);
-    }
-
-    private function makeStandardWord($sTerm)
-    {
-        return trim($this->oTransliterator->transliterate(' '.$sTerm.' '));
-    }
-
-
-    public function tokensForSpecialTerm($sTerm)
-    {
-        $aResults = array();
-
-        $sSQL = "SELECT word_id, info->>'class' as class, info->>'type' as type ";
-        $sSQL .= '   FROM word WHERE word_token = :term and type = \'S\'';
-
-        Debug::printVar('Term', $sTerm);
-        Debug::printSQL($sSQL);
-        $aSearchWords = $this->oDB->getAll($sSQL, array(':term' => $this->makeStandardWord($sTerm)));
-
-        Debug::printVar('Results', $aSearchWords);
-
-        foreach ($aSearchWords as $aSearchTerm) {
-            $aResults[] = new \Nominatim\Token\SpecialTerm(
-                $aSearchTerm['word_id'],
-                $aSearchTerm['class'],
-                $aSearchTerm['type'],
-                \Nominatim\Operator::TYPE
-            );
-        }
-
-        Debug::printVar('Special term tokens', $aResults);
-
-        return $aResults;
-    }
-
-
-    public function extractTokensFromPhrases(&$aPhrases)
-    {
-        $sNormQuery = '';
-        $aWordLists = array();
-        $aTokens = array();
-        foreach ($aPhrases as $iPhrase => $oPhrase) {
-            $sNormQuery .= ','.$this->normalizeString($oPhrase->getPhrase());
-            $sPhrase = $this->makeStandardWord($oPhrase->getPhrase());
-            Debug::printVar('Phrase', $sPhrase);
-            if (strlen($sPhrase) > 0) {
-                $aWords = explode(' ', $sPhrase);
-                Tokenizer::addTokens($aTokens, $aWords);
-                $aWordLists[] = $aWords;
-            } else {
-                $aWordLists[] = array();
-            }
-        }
-
-        Debug::printVar('Tokens', $aTokens);
-        Debug::printVar('WordLists', $aWordLists);
-
-        $oValidTokens = $this->computeValidTokens($aTokens, $sNormQuery);
-
-        foreach ($aPhrases as $iPhrase => $oPhrase) {
-            $oPhrase->computeWordSets($aWordLists[$iPhrase], $oValidTokens);
-        }
-
-        return $oValidTokens;
-    }
-
-
-    private function computeValidTokens($aTokens, $sNormQuery)
-    {
-        $oValidTokens = new TokenList();
-
-        if (!empty($aTokens)) {
-            $this->addTokensFromDB($oValidTokens, $aTokens, $sNormQuery);
-
-            // Try more interpretations for Tokens that could not be matched.
-            foreach ($aTokens as $sToken) {
-                if ($sToken[0] != ' ' && !$oValidTokens->contains($sToken)) {
-                    if (preg_match('/^([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) {
-                        // US ZIP+4 codes - merge in the 5-digit ZIP code
-                        $oValidTokens->addToken(
-                            $sToken,
-                            new Token\Postcode(null, $aData[1], 'us')
-                        );
-                    } elseif (preg_match('/^[0-9]+$/', $sToken)) {
-                        // Unknown single word token with a number.
-                        // Assume it is a house number.
-                        $oValidTokens->addToken(
-                            $sToken,
-                            new Token\HouseNumber(null, trim($sToken))
-                        );
-                    }
-                }
-            }
-        }
-
-        return $oValidTokens;
-    }
-
-
-    private function addTokensFromDB(&$oValidTokens, $aTokens, $sNormQuery)
-    {
-        // Check which tokens we have, get the ID numbers
-        $sSQL = 'SELECT word_id, word_token, type, word,';
-        $sSQL .= "      info->>'op' as operator,";
-        $sSQL .= "      info->>'class' as class, info->>'type' as ctype,";
-        $sSQL .= "      info->>'count' as count";
-        $sSQL .= ' FROM word WHERE word_token in (';
-        $sSQL .= join(',', $this->oDB->getDBQuotedList($aTokens)).')';
-
-        Debug::printSQL($sSQL);
-
-        $aDBWords = $this->oDB->getAll($sSQL, null, 'Could not get word tokens.');
-
-        foreach ($aDBWords as $aWord) {
-            $iId = (int) $aWord['word_id'];
-            $sTok = $aWord['word_token'];
-
-            switch ($aWord['type']) {
-                case 'C':  // country name tokens
-                    if ($aWord['word'] !== null
-                        && (!$this->aCountryRestriction
-                            || in_array($aWord['word'], $this->aCountryRestriction))
-                    ) {
-                        $oValidTokens->addToken(
-                            $sTok,
-                            new Token\Country($iId, $aWord['word'])
-                        );
-                    }
-                    break;
-                case 'H':  // house number tokens
-                    $oValidTokens->addToken($sTok, new Token\HouseNumber($iId, $aWord['word_token']));
-                    break;
-                case 'P':  // postcode tokens
-                    // Postcodes are not normalized, so they may have content
-                    // that makes SQL injection possible. Reject postcodes
-                    // that would need special escaping.
-                    if ($aWord['word'] !== null
-                        && pg_escape_string($aWord['word']) == $aWord['word']
-                    ) {
-                        $sNormPostcode = $this->normalizeString($aWord['word']);
-                        if (strpos($sNormQuery, $sNormPostcode) !== false) {
-                            $oValidTokens->addToken(
-                                $sTok,
-                                new Token\Postcode($iId, $aWord['word'], null)
-                            );
-                        }
-                    }
-                    break;
-                case 'S':  // tokens for classification terms (special phrases)
-                    if ($aWord['class'] !== null && $aWord['ctype'] !== null) {
-                        $oValidTokens->addToken($sTok, new Token\SpecialTerm(
-                            $iId,
-                            $aWord['class'],
-                            $aWord['ctype'],
-                            (isset($aWord['operator'])) ? Operator::NEAR : Operator::NONE
-                        ));
-                    }
-                    break;
-                case 'W': // full-word tokens
-                    $oValidTokens->addToken($sTok, new Token\Word(
-                        $iId,
-                        (int) $aWord['count'],
-                        substr_count($aWord['word_token'], ' ')
-                    ));
-                    break;
-                case 'w':  // partial word terms
-                    $oValidTokens->addToken($sTok, new Token\Partial(
-                        $iId,
-                        $aWord['word_token'],
-                        (int) $aWord['count']
-                    ));
-                    break;
-                default:
-                    break;
-            }
-        }
-    }
-
-
-    /**
-     * Add the tokens from this phrase to the given list of tokens.
-     *
-     * @param string[] $aTokens List of tokens to append.
-     *
-     * @return void
-     */
-    private static function addTokens(&$aTokens, $aWords)
-    {
-        $iNumWords = count($aWords);
-
-        for ($i = 0; $i < $iNumWords; $i++) {
-            $sPhrase = $aWords[$i];
-            $aTokens[$sPhrase] = $sPhrase;
-
-            for ($j = $i + 1; $j < $iNumWords; $j++) {
-                $sPhrase .= ' '.$aWords[$j];
-                $aTokens[$sPhrase] = $sPhrase;
-            }
-        }
-    }
-}

lib-php/tokenizer/legacy_tokenizer.php

@@ -1,266 +0,0 @@
-<?php
-
-namespace Nominatim;
-
-class Tokenizer
-{
-    private $oDB;
-
-    private $oNormalizer = null;
-    private $aCountryRestriction = null;
-
-    public function __construct(&$oDB)
-    {
-        $this->oDB =& $oDB;
-        $this->oNormalizer = \Transliterator::createFromRules(CONST_Term_Normalization_Rules);
-    }
-
-    public function checkStatus()
-    {
-        $sStandardWord = $this->oDB->getOne("SELECT make_standard_name('a')");
-        if ($sStandardWord === false) {
-            throw new \Exception('Module failed', 701);
-        }
-
-        if ($sStandardWord != 'a') {
-            throw new \Exception('Module call failed', 702);
-        }
-
-        $sSQL = "SELECT word_id FROM word WHERE word_token IN (' a')";
-        $iWordID = $this->oDB->getOne($sSQL);
-        if ($iWordID === false) {
-            throw new \Exception('Query failed', 703);
-        }
-        if (!$iWordID) {
-            throw new \Exception('No value', 704);
-        }
-    }
-
-
-    public function setCountryRestriction($aCountries)
-    {
-        $this->aCountryRestriction = $aCountries;
-    }
-
-
-    public function normalizeString($sTerm)
-    {
-        if ($this->oNormalizer === null) {
-            return $sTerm;
-        }
-
-        return $this->oNormalizer->transliterate($sTerm);
-    }
-
-
-    public function tokensForSpecialTerm($sTerm)
-    {
-        $aResults = array();
-
-        $sSQL = 'SELECT word_id, class, type FROM word ';
-        $sSQL .= '   WHERE word_token = \' \' || make_standard_name(:term)';
-        $sSQL .= '   AND class is not null AND class not in (\'place\')';
-
-        Debug::printVar('Term', $sTerm);
-        Debug::printSQL($sSQL);
-        $aSearchWords = $this->oDB->getAll($sSQL, array(':term' => $sTerm));
-
-        Debug::printVar('Results', $aSearchWords);
-
-        foreach ($aSearchWords as $aSearchTerm) {
-            $aResults[] = new \Nominatim\Token\SpecialTerm(
-                $aSearchTerm['word_id'],
-                $aSearchTerm['class'],
-                $aSearchTerm['type'],
-                \Nominatim\Operator::TYPE
-            );
-        }
-
-        Debug::printVar('Special term tokens', $aResults);
-
-        return $aResults;
-    }
-
-
-    public function extractTokensFromPhrases(&$aPhrases)
-    {
-        // First get the normalized version of all phrases
-        $sNormQuery = '';
-        $sSQL = 'SELECT ';
-        $aParams = array();
-        foreach ($aPhrases as $iPhrase => $oPhrase) {
-            $sNormQuery .= ','.$this->normalizeString($oPhrase->getPhrase());
-            $sSQL .= 'make_standard_name(:' .$iPhrase.') as p'.$iPhrase.',';
-            $aParams[':'.$iPhrase] = $oPhrase->getPhrase();
-        }
-        $sSQL = substr($sSQL, 0, -1);
-
-        Debug::printSQL($sSQL);
-        Debug::printVar('SQL parameters', $aParams);
-
-        $aNormPhrases = $this->oDB->getRow($sSQL, $aParams);
-
-        Debug::printVar('SQL result', $aNormPhrases);
-
-        // now compute all possible tokens
-        $aWordLists = array();
-        $aTokens = array();
-        foreach ($aNormPhrases as $sPhrase) {
-            if (strlen($sPhrase) > 0) {
-                $aWords = explode(' ', $sPhrase);
-                Tokenizer::addTokens($aTokens, $aWords);
-                $aWordLists[] = $aWords;
-            } else {
-                $aWordLists[] = array();
-            }
-        }
-
-        Debug::printVar('Tokens', $aTokens);
-        Debug::printVar('WordLists', $aWordLists);
-
-        $oValidTokens = $this->computeValidTokens($aTokens, $sNormQuery);
-
-        foreach ($aPhrases as $iPhrase => $oPhrase) {
-            $oPhrase->computeWordSets($aWordLists[$iPhrase], $oValidTokens);
-        }
-
-        return $oValidTokens;
-    }
-
-
-    private function computeValidTokens($aTokens, $sNormQuery)
-    {
-        $oValidTokens = new TokenList();
-
-        if (!empty($aTokens)) {
-            $this->addTokensFromDB($oValidTokens, $aTokens, $sNormQuery);
-
-            // Try more interpretations for Tokens that could not be matched.
-            foreach ($aTokens as $sToken) {
-                if ($sToken[0] != ' ' && !$oValidTokens->contains($sToken)) {
-                    if (preg_match('/^([0-9]{5}) [0-9]{4}$/', $sToken, $aData)) {
-                        // US ZIP+4 codes - merge in the 5-digit ZIP code
-                        $oValidTokens->addToken(
-                            $sToken,
-                            new Token\Postcode(null, $aData[1], 'us')
-                        );
-                    } elseif (preg_match('/^[0-9]+$/', $sToken)) {
-                        // Unknown single word token with a number.
-                        // Assume it is a house number.
-                        $oValidTokens->addToken(
-                            $sToken,
-                            new Token\HouseNumber(null, trim($sToken))
-                        );
-                    }
-                }
-            }
-        }
-
-        return $oValidTokens;
-    }
-
-
-    private function addTokensFromDB(&$oValidTokens, $aTokens, $sNormQuery)
-    {
-        // Check which tokens we have, get the ID numbers
-        $sSQL = 'SELECT word_id, word_token, word, class, type, country_code,';
-        $sSQL .= ' operator, coalesce(search_name_count, 0) as count';
-        $sSQL .= ' FROM word WHERE word_token in (';
-        $sSQL .= join(',', $this->oDB->getDBQuotedList($aTokens)).')';
-
-        Debug::printSQL($sSQL);
-
-        $aDBWords = $this->oDB->getAll($sSQL, null, 'Could not get word tokens.');
-
-        foreach ($aDBWords as $aWord) {
-            $oToken = null;
-            $iId = (int) $aWord['word_id'];
-
-            if ($aWord['class']) {
-                // Special terms need to appear in their normalized form.
-                // (postcodes are not normalized in the word table)
-                $sNormWord = $this->normalizeString($aWord['word']);
-                if ($aWord['word'] && strpos($sNormQuery, $sNormWord) === false) {
-                    continue;
-                }
-
-                if ($aWord['class'] == 'place' && $aWord['type'] == 'house') {
-                    $oToken = new Token\HouseNumber($iId, trim($aWord['word_token']));
-                } elseif ($aWord['class'] == 'place' && $aWord['type'] == 'postcode') {
-                    if ($aWord['word']
-                        && pg_escape_string($aWord['word']) == $aWord['word']
-                    ) {
-                        $oToken = new Token\Postcode(
-                            $iId,
-                            $aWord['word'],
-                            $aWord['country_code']
-                        );
-                    }
-                } else {
-                    // near and in operator the same at the moment
-                    $oToken = new Token\SpecialTerm(
-                        $iId,
-                        $aWord['class'],
-                        $aWord['type'],
-                        $aWord['operator'] ? Operator::NEAR : Operator::NONE
-                    );
-                }
-            } elseif ($aWord['country_code']) {
-                // Filter country tokens that do not match restricted countries.
-                if (!$this->aCountryRestriction
-                    || in_array($aWord['country_code'], $this->aCountryRestriction)
-                ) {
-                    $oToken = new Token\Country($iId, $aWord['country_code']);
-                }
-            } elseif ($aWord['word_token'][0] == ' ') {
-                $oToken = new Token\Word(
-                    $iId,
-                    (int) $aWord['count'],
-                    substr_count($aWord['word_token'], ' ')
-                );
-            // For backward compatibility: ignore all partial tokens with more
-            // than one word.
-            } elseif (strpos($aWord['word_token'], ' ') === false) {
-                $oToken = new Token\Partial(
-                    $iId,
-                    $aWord['word_token'],
-                    (int) $aWord['count']
-                );
-            }
-
-            if ($oToken) {
-                // remove any leading spaces
-                if ($aWord['word_token'][0] == ' ') {
-                    $oValidTokens->addToken(substr($aWord['word_token'], 1), $oToken);
-                } else {
-                    $oValidTokens->addToken($aWord['word_token'], $oToken);
-                }
-            }
-        }
-    }
-
-
-    /**
-     * Add the tokens from this phrase to the given list of tokens.
-     *
-     * @param string[] $aTokens List of tokens to append.
-     *
-     * @return void
-     */
-    private static function addTokens(&$aTokens, $aWords)
-    {
-        $iNumWords = count($aWords);
-
-        for ($i = 0; $i < $iNumWords; $i++) {
-            $sPhrase = $aWords[$i];
-            $aTokens[' '.$sPhrase] = ' '.$sPhrase;
-            $aTokens[$sPhrase] = $sPhrase;
-
-            for ($j = $i + 1; $j < $iNumWords; $j++) {
-                $sPhrase .= ' '.$aWords[$j];
-                $aTokens[' '.$sPhrase] = ' '.$sPhrase;
-                $aTokens[$sPhrase] = $sPhrase;
-            }
-        }
-    }
-}

lib-php/website/details.php

@@ -83,7 +83,7 @@ if ($sOsmType && $iOsmId > 0) {
     }
 
     if ($sPlaceId === false) {
-        throw new \Exception('No place with that OSM ID found.', 404);
+        throw new Exception('No place with that OSM ID found.', 404);
     }
 } else {
     if ($sPlaceId === false) {
@@ -95,21 +95,20 @@ $iPlaceID = (int)$sPlaceId;
 
 if (CONST_Use_US_Tiger_Data) {
     $iParentPlaceID = $oDB->getOne('SELECT parent_place_id FROM location_property_tiger WHERE place_id = '.$iPlaceID);
-    if ($iParentPlaceID) {
-        $iPlaceID = $iParentPlaceID;
-    }
+    if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
 }
 
 // interpolated house numbers
 $iParentPlaceID = $oDB->getOne('SELECT parent_place_id FROM location_property_osmline WHERE place_id = '.$iPlaceID);
-if ($iParentPlaceID) {
-    $iPlaceID = $iParentPlaceID;
-}
+if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
 
 // artificial postcodes
 $iParentPlaceID = $oDB->getOne('SELECT parent_place_id FROM location_postcode WHERE place_id = '.$iPlaceID);
-if ($iParentPlaceID) {
-    $iPlaceID = $iParentPlaceID;
+if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+
+if (CONST_Use_Aux_Location_data) {
+    $iParentPlaceID = $oDB->getOne('SELECT parent_place_id FROM location_property_aux WHERE place_id = '.$iPlaceID);
+    if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
 }
 
 $hLog = logStart($oDB, 'details', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
@@ -146,7 +145,7 @@ $sSQL .= " WHERE place_id = $iPlaceID";
 $aPointDetails = $oDB->getRow($sSQL, null, 'Could not get details of place object.');
 
 if (!$aPointDetails) {
-    throw new \Exception('No place with that place ID found.', 404);
+    throw new Exception('No place with that place ID found.', 404);
 }
 
 $aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber'];

lib-php/website/lookup.php

@@ -35,10 +35,8 @@ if (count($aOsmIds) > CONST_Places_Max_ID_count) {
 
 foreach ($aOsmIds as $sItem) {
     // Skip empty sItem
-    if (empty($sItem)) {
-        continue;
-    }
-
+    if (empty($sItem)) continue;
+    
     $sType = $sItem[0];
     $iId = (int) substr($sItem, 1);
     if ($iId > 0 && ($sType == 'N' || $sType == 'W' || $sType == 'R')) {
@@ -50,9 +48,7 @@ foreach ($aOsmIds as $sItem) {
             // key names
             $oResult = $oPlace;
             unset($oResult['aAddress']);
-            if (isset($oPlace['aAddress'])) {
-                $oResult['address'] = $oPlace['aAddress'];
-            }
+            if (isset($oPlace['aAddress'])) $oResult['address'] = $oPlace['aAddress'];
             if ($sOutputFormat != 'geocodejson') {
                 unset($oResult['langaddress']);
                 $oResult['name'] = $oPlace['langaddress'];
@@ -75,9 +71,7 @@ foreach ($aOsmIds as $sItem) {
 }
 
 
-if (CONST_Debug) {
-    exit;
-}
+if (CONST_Debug) exit;
 
 $sXmlRootTag = 'lookupresults';
 $sQuery = join(',', $aCleanedQueryParts);

lib-php/website/polygons.php

@@ -30,12 +30,8 @@ while ($iTotalBroken && empty($aPolygons)) {
         $iDays++;
     }
 
-    if ($bReduced) {
-        $aWhere[] = "errormessage like 'Area reduced%'";
-    }
-    if ($sClass) {
-        $sWhere[] = "class = '".pg_escape_string($sClass)."'";
-    }
+    if ($bReduced) $aWhere[] = "errormessage like 'Area reduced%'";
+    if ($sClass) $sWhere[] = "class = '".pg_escape_string($sClass)."'";
 
     if (!empty($aWhere)) {
         $sSQL .= ' WHERE '.join(' and ', $aWhere);

lib-php/website/reverse-only-search.php

@@ -1,12 +0,0 @@
-<?php
-
-require_once(CONST_LibDir.'/init-website.php');
-require_once(CONST_LibDir.'/ParameterParser.php');
-
-$oParams = new Nominatim\ParameterParser();
-
-// Format for output
-$sOutputFormat = $oParams->getSet('format', array('xml', 'json', 'jsonv2', 'geojson', 'geocodejson'), 'jsonv2');
-set_exception_handler_by_format($sOutputFormat);
-
-throw new Exception('Reverse-only import does not support forward searching.', 404);

lib-php/website/search.php

@@ -77,9 +77,7 @@ if (isset($_SERVER['REQUEST_SCHEME'])
     $sMoreURL = '/search.php?'.http_build_query($aMoreParams);
 }
 
-if (CONST_Debug) {
-    exit;
-}
+if (CONST_Debug) exit;
 
 $sOutputTemplate = ($sOutputFormat == 'jsonv2') ? 'json' : $sOutputFormat;
 include(CONST_LibDir.'/template/search-'.$sOutputTemplate.'.php');

lib-sql/aux_tables.sql

@@ -0,0 +1,6 @@
+CREATE TABLE location_property_aux () INHERITS (location_property);
+CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
+CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
+CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
+GRANT SELECT ON location_property_aux TO "{www-user}";
+

lib-sql/functions.sql

@@ -1,4 +1,5 @@
 {% include('functions/utils.sql') %}
+{% include('functions/normalization.sql') %}
 {% include('functions/ranking.sql') %}
 {% include('functions/importance.sql') %}
 {% include('functions/address_lookup.sql') %}

lib-sql/functions/address_lookup.sql

@@ -135,6 +135,20 @@ BEGIN
   END IF;
   {% endif %}
 
+  -- then additional data
+  {% if config.get_bool('USE_AUX_LOCATION_DATA') %}
+  IF place IS NULL THEN
+    SELECT parent_place_id as place_id, 'us' as country_code,
+           housenumber, postcode,
+           'place' as class, 'house' as type,
+           null as name, null as address,
+           centroid
+      INTO place
+      FROM location_property_aux
+      WHERE place_id = in_place_id;
+  END IF;
+  {% endif %}
+
   -- postcode table
   IF place IS NULL THEN
     SELECT parent_place_id as place_id, country_code,

lib-sql/functions/aux_property.sql

@@ -0,0 +1,53 @@
+-- Functions for adding external data (currently unused).
+
+CREATE OR REPLACE FUNCTION aux_create_property(pointgeo GEOMETRY, in_housenumber TEXT,
+                                               in_street TEXT, in_isin TEXT,
+                                               in_postcode TEXT, in_countrycode char(2))
+  RETURNS INTEGER
+  AS $$
+DECLARE
+
+  newpoints INTEGER;
+  place_centroid GEOMETRY;
+  out_partition INTEGER;
+  out_parent_place_id BIGINT;
+  location RECORD;
+  address_street_word_ids INTEGER[];
+  out_postcode TEXT;
+
+BEGIN
+
+  place_centroid := ST_Centroid(pointgeo);
+  out_partition := get_partition(in_countrycode);
+  out_parent_place_id := null;
+
+  address_street_word_ids := word_ids_from_name(in_street);
+  IF address_street_word_ids IS NOT NULL THEN
+    out_parent_place_id := getNearestNamedRoadPlaceId(out_partition, place_centroid,
+                                                      address_street_word_ids);
+  END IF;
+
+  IF out_parent_place_id IS NULL THEN
+    SELECT getNearestRoadPlaceId(out_partition, place_centroid)
+      INTO out_parent_place_id;
+    END LOOP;
+  END IF;
+
+  out_postcode := in_postcode;
+  IF out_postcode IS NULL THEN
+    SELECT postcode from placex where place_id = out_parent_place_id INTO out_postcode;
+  END IF;
+  -- XXX look into postcode table
+
+  newpoints := 0;
+  insert into location_property_aux (place_id, partition, parent_place_id,
+                                     housenumber, postcode, centroid)
+    values (nextval('seq_place'), out_partition, out_parent_place_id,
+            in_housenumber, out_postcode, place_centroid);
+  newpoints := newpoints + 1;
+
+  RETURN newpoints;
+END;
+$$
+LANGUAGE plpgsql;
+

lib-sql/functions/interpolation.sql

@@ -12,47 +12,39 @@ $$
 LANGUAGE plpgsql IMMUTABLE;
 
 
-CREATE OR REPLACE FUNCTION get_interpolation_address(in_address HSTORE, wayid BIGINT)
-RETURNS HSTORE
-  AS $$
-DECLARE
-  location RECORD;
-  waynodes BIGINT[];
-BEGIN
-  IF akeys(in_address) != ARRAY['interpolation'] THEN
-    RETURN in_address;
-  END IF;
-
-  SELECT nodes INTO waynodes FROM planet_osm_ways WHERE id = wayid;
-  FOR location IN
-    SELECT placex.address, placex.osm_id FROM placex
-     WHERE osm_type = 'N' and osm_id = ANY(waynodes)
-           and placex.address is not null
-           and (placex.address ? 'street' or placex.address ? 'place')
-           and indexed_status < 100
-  LOOP
-    -- mark it as a derived address
-    RETURN location.address || in_address || hstore('_inherited', '');
-  END LOOP;
-
-  RETURN in_address;
-END;
-$$
-LANGUAGE plpgsql STABLE;
-
-
-
 -- find the parent road of the cut road parts
-CREATE OR REPLACE FUNCTION get_interpolation_parent(street INTEGER[], place INTEGER[],
-                                                    partition SMALLINT,
+CREATE OR REPLACE FUNCTION get_interpolation_parent(wayid BIGINT, street TEXT,
+                                                    place TEXT, partition SMALLINT,
                                                     centroid GEOMETRY, geom GEOMETRY)
   RETURNS BIGINT
   AS $$
 DECLARE
+  addr_street TEXT;
+  addr_place TEXT;
   parent_place_id BIGINT;
+
+  waynodes BIGINT[];
+
   location RECORD;
 BEGIN
-  parent_place_id := find_parent_for_address(street, place, partition, centroid);
+  addr_street = street;
+  addr_place = place;
+
+  IF addr_street is null and addr_place is null THEN
+    select nodes from planet_osm_ways where id = wayid INTO waynodes;
+    FOR location IN SELECT placex.address from placex
+                    where osm_type = 'N' and osm_id = ANY(waynodes)
+                          and placex.address is not null
+                          and (placex.address ? 'street' or placex.address ? 'place')
+                          and indexed_status < 100
+                    limit 1 LOOP
+      addr_street = location.address->'street';
+      addr_place = location.address->'place';
+    END LOOP;
+  END IF;
+
+  parent_place_id := find_parent_for_address(addr_street, addr_place,
+                                             partition, centroid);
 
   IF parent_place_id is null THEN
     FOR location IN SELECT place_id FROM placex
@@ -155,15 +147,15 @@ BEGIN
   NEW.interpolationtype = NEW.address->'interpolation';
 
   place_centroid := ST_PointOnSurface(NEW.linegeo);
-  NEW.parent_place_id = get_interpolation_parent(token_addr_street_match_tokens(NEW.token_info),
-                                                 token_addr_place_match_tokens(NEW.token_info),
+  NEW.parent_place_id = get_interpolation_parent(NEW.osm_id, NEW.address->'street',
+                                                 NEW.address->'place',
                                                  NEW.partition, place_centroid, NEW.linegeo);
 
-  interpol_postcode := token_normalized_postcode(NEW.address->'postcode');
-
-  NEW.token_info := token_strip_info(NEW.token_info);
-  IF NEW.address ? '_inherited' THEN
-    NEW.address := hstore('interpolation', NEW.interpolationtype);
+  IF NEW.address is not NULL AND NEW.address ? 'postcode' AND NEW.address->'postcode' not similar to '%(,|;)%' THEN
+    interpol_postcode := NEW.address->'postcode';
+    housenum := getorcreate_postcode_id(NEW.address->'postcode');
+  ELSE
+    interpol_postcode := NULL;
   END IF;
 
   -- if the line was newly inserted, split the line as necessary
@@ -210,13 +202,12 @@ BEGIN
 
             -- determine postcode
             postcode := coalesce(interpol_postcode,
-                                 token_normalized_postcode(prevnode.address->'postcode'),
-                                 token_normalized_postcode(nextnode.address->'postcode'),
+                                 prevnode.address->'postcode',
+                                 nextnode.address->'postcode',
                                  postcode);
 
             IF postcode is NULL THEN
-                SELECT token_normalized_postcode(placex.postcode)
-                  FROM placex WHERE place_id = NEW.parent_place_id INTO postcode;
+                SELECT placex.postcode FROM placex WHERE place_id = NEW.parent_place_id INTO postcode;
             END IF;
             IF postcode is NULL THEN
                 postcode := get_nearest_postcode(NEW.country_code, nextnode.geometry);
@@ -226,7 +217,7 @@ BEGIN
                 NEW.startnumber := startnumber;
                 NEW.endnumber := endnumber;
                 NEW.linegeo := sectiongeo;
-                NEW.postcode := postcode;
+                NEW.postcode := upper(trim(postcode));
              ELSE
               insert into location_property_osmline
                      (linegeo, partition, osm_id, parent_place_id,

lib-sql/functions/normalization.sql

@@ -0,0 +1,545 @@
+-- Functions for term normalisation and access to the 'word' table.
+
+CREATE OR REPLACE FUNCTION transliteration(text) RETURNS text
+  AS '{{ modulepath }}/nominatim.so', 'transliteration'
+LANGUAGE c IMMUTABLE STRICT;
+
+
+CREATE OR REPLACE FUNCTION gettokenstring(text) RETURNS text
+  AS '{{ modulepath }}/nominatim.so', 'gettokenstring'
+LANGUAGE c IMMUTABLE STRICT;
+
+
+CREATE OR REPLACE FUNCTION make_standard_name(name TEXT) RETURNS TEXT
+  AS $$
+DECLARE
+  o TEXT;
+BEGIN
+  o := public.gettokenstring(public.transliteration(name));
+  RETURN trim(substr(o,1,length(o)));
+END;
+$$
+LANGUAGE plpgsql IMMUTABLE;
+
+-- returns NULL if the word is too common
+CREATE OR REPLACE FUNCTION getorcreate_word_id(lookup_word TEXT) 
+  RETURNS INTEGER
+  AS $$
+DECLARE
+  lookup_token TEXT;
+  return_word_id INTEGER;
+  count INTEGER;
+BEGIN
+  lookup_token := trim(lookup_word);
+  SELECT min(word_id), max(search_name_count) FROM word
+    WHERE word_token = lookup_token and class is null and type is null
+    INTO return_word_id, count;
+  IF return_word_id IS NULL THEN
+    return_word_id := nextval('seq_word');
+    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, null, null, 0);
+  ELSE
+    IF count > get_maxwordfreq() THEN
+      return_word_id := NULL;
+    END IF;
+  END IF;
+  RETURN return_word_id;
+END;
+$$
+LANGUAGE plpgsql;
+
+-- Create housenumber tokens from an OSM addr:housenumber.
+-- The housnumber is split at comma and semicolon as necessary.
+-- The function returns the normalized form of the housenumber suitable
+-- for comparison.
+CREATE OR REPLACE FUNCTION create_housenumber_id(housenumber TEXT)
+  RETURNS TEXT
+  AS $$
+DECLARE
+  normtext TEXT;
+BEGIN
+  SELECT array_to_string(array_agg(trans), ';')
+    INTO normtext
+    FROM (SELECT lookup_word as trans, getorcreate_housenumber_id(lookup_word)
+          FROM (SELECT make_standard_name(h) as lookup_word
+                FROM regexp_split_to_table(housenumber, '[,;]') h) x) y;
+
+  return normtext;
+END;
+$$ LANGUAGE plpgsql STABLE STRICT;
+
+CREATE OR REPLACE FUNCTION getorcreate_housenumber_id(lookup_word TEXT)
+  RETURNS INTEGER
+  AS $$
+DECLARE
+  lookup_token TEXT;
+  return_word_id INTEGER;
+BEGIN
+  lookup_token := ' ' || trim(lookup_word);
+  SELECT min(word_id) FROM word
+    WHERE word_token = lookup_token and class='place' and type='house'
+    INTO return_word_id;
+  IF return_word_id IS NULL THEN
+    return_word_id := nextval('seq_word');
+    INSERT INTO word VALUES (return_word_id, lookup_token, null,
+                             'place', 'house', null, 0);
+  END IF;
+  RETURN return_word_id;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION getorcreate_postcode_id(postcode TEXT)
+  RETURNS INTEGER
+  AS $$
+DECLARE
+  lookup_token TEXT;
+  lookup_word TEXT;
+  return_word_id INTEGER;
+BEGIN
+  lookup_word := upper(trim(postcode));
+  lookup_token := ' ' || make_standard_name(lookup_word);
+  SELECT min(word_id) FROM word
+    WHERE word_token = lookup_token and word = lookup_word
+          and class='place' and type='postcode'
+    INTO return_word_id;
+  IF return_word_id IS NULL THEN
+    return_word_id := nextval('seq_word');
+    INSERT INTO word VALUES (return_word_id, lookup_token, lookup_word,
+                             'place', 'postcode', null, 0);
+  END IF;
+  RETURN return_word_id;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION getorcreate_country(lookup_word TEXT,
+                                               lookup_country_code varchar(2))
+  RETURNS INTEGER
+  AS $$
+DECLARE
+  lookup_token TEXT;
+  return_word_id INTEGER;
+BEGIN
+  lookup_token := ' '||trim(lookup_word);
+  SELECT min(word_id) FROM word
+    WHERE word_token = lookup_token and country_code=lookup_country_code
+    INTO return_word_id;
+  IF return_word_id IS NULL THEN
+    return_word_id := nextval('seq_word');
+    INSERT INTO word VALUES (return_word_id, lookup_token, null,
+                             null, null, lookup_country_code, 0);
+  END IF;
+  RETURN return_word_id;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION getorcreate_amenity(lookup_word TEXT, normalized_word TEXT,
+                                               lookup_class text, lookup_type text)
+  RETURNS INTEGER
+  AS $$
+DECLARE
+  lookup_token TEXT;
+  return_word_id INTEGER;
+BEGIN
+  lookup_token := ' '||trim(lookup_word);
+  SELECT min(word_id) FROM word
+  WHERE word_token = lookup_token and word = normalized_word
+        and class = lookup_class and type = lookup_type
+  INTO return_word_id;
+  IF return_word_id IS NULL THEN
+    return_word_id := nextval('seq_word');
+    INSERT INTO word VALUES (return_word_id, lookup_token, normalized_word,
+                             lookup_class, lookup_type, null, 0);
+  END IF;
+  RETURN return_word_id;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION getorcreate_amenityoperator(lookup_word TEXT,
+                                                       normalized_word TEXT,
+                                                       lookup_class text,
+                                                       lookup_type text,
+                                                       op text)
+  RETURNS INTEGER
+  AS $$
+DECLARE
+  lookup_token TEXT;
+  return_word_id INTEGER;
+BEGIN
+  lookup_token := ' '||trim(lookup_word);
+  SELECT min(word_id) FROM word
+  WHERE word_token = lookup_token and word = normalized_word
+        and class = lookup_class and type = lookup_type and operator = op
+  INTO return_word_id;
+  IF return_word_id IS NULL THEN
+    return_word_id := nextval('seq_word');
+    INSERT INTO word VALUES (return_word_id, lookup_token, normalized_word,
+                             lookup_class, lookup_type, null, 0, op);
+  END IF;
+  RETURN return_word_id;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION getorcreate_name_id(lookup_word TEXT, src_word TEXT)
+  RETURNS INTEGER
+  AS $$
+DECLARE
+  lookup_token TEXT;
+  nospace_lookup_token TEXT;
+  return_word_id INTEGER;
+BEGIN
+  lookup_token := ' '||trim(lookup_word);
+  SELECT min(word_id) FROM word
+  WHERE word_token = lookup_token and class is null and type is null
+  INTO return_word_id;
+  IF return_word_id IS NULL THEN
+    return_word_id := nextval('seq_word');
+    INSERT INTO word VALUES (return_word_id, lookup_token, src_word,
+                             null, null, null, 0);
+  END IF;
+  RETURN return_word_id;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION getorcreate_name_id(lookup_word TEXT)
+  RETURNS INTEGER
+  AS $$
+DECLARE
+BEGIN
+  RETURN getorcreate_name_id(lookup_word, '');
+END;
+$$
+LANGUAGE plpgsql;
+
+-- Normalize a string and lookup its word ids (partial words).
+CREATE OR REPLACE FUNCTION addr_ids_from_name(lookup_word TEXT)
+  RETURNS INTEGER[]
+  AS $$
+DECLARE
+  words TEXT[];
+  id INTEGER;
+  return_word_id INTEGER[];
+  word_ids INTEGER[];
+  j INTEGER;
+BEGIN
+  words := string_to_array(make_standard_name(lookup_word), ' ');
+  IF array_upper(words, 1) IS NOT NULL THEN
+    FOR j IN 1..array_upper(words, 1) LOOP
+      IF (words[j] != '') THEN
+        SELECT array_agg(word_id) INTO word_ids
+          FROM word
+         WHERE word_token = words[j] and class is null and type is null;
+
+        IF word_ids IS NULL THEN
+          id := nextval('seq_word');
+          INSERT INTO word VALUES (id, words[j], null, null, null, null, 0);
+          return_word_id := return_word_id || id;
+        ELSE
+          return_word_id := array_merge(return_word_id, word_ids);
+        END IF;
+      END IF;
+    END LOOP;
+  END IF;
+
+  RETURN return_word_id;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+-- Normalize a string and look up its name ids (full words).
+CREATE OR REPLACE FUNCTION word_ids_from_name(lookup_word TEXT)
+  RETURNS INTEGER[]
+  AS $$
+DECLARE
+  lookup_token TEXT;
+  return_word_ids INTEGER[];
+BEGIN
+  lookup_token := ' '|| make_standard_name(lookup_word);
+  SELECT array_agg(word_id) FROM word
+    WHERE word_token = lookup_token and class is null and type is null
+    INTO return_word_ids;
+  RETURN return_word_ids;
+END;
+$$
+LANGUAGE plpgsql STABLE STRICT;
+
+
+CREATE OR REPLACE FUNCTION create_country(src HSTORE, country_code varchar(2))
+  RETURNS VOID
+  AS $$
+DECLARE
+  s TEXT;
+  w INTEGER;
+  words TEXT[];
+  item RECORD;
+  j INTEGER;
+BEGIN
+  FOR item IN SELECT (each(src)).* LOOP
+
+    s := make_standard_name(item.value);
+    w := getorcreate_country(s, country_code);
+
+    words := regexp_split_to_array(item.value, E'[,;()]');
+    IF array_upper(words, 1) != 1 THEN
+      FOR j IN 1..array_upper(words, 1) LOOP
+        s := make_standard_name(words[j]);
+        IF s != '' THEN
+          w := getorcreate_country(s, country_code);
+        END IF;
+      END LOOP;
+    END IF;
+  END LOOP;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION make_keywords(src HSTORE)
+  RETURNS INTEGER[]
+  AS $$
+DECLARE
+  result INTEGER[];
+  s TEXT;
+  w INTEGER;
+  words TEXT[];
+  item RECORD;
+  j INTEGER;
+BEGIN
+  result := '{}'::INTEGER[];
+
+  FOR item IN SELECT (each(src)).* LOOP
+
+    s := make_standard_name(item.value);
+    w := getorcreate_name_id(s, item.value);
+
+    IF not(ARRAY[w] <@ result) THEN
+      result := result || w;
+    END IF;
+
+    w := getorcreate_word_id(s);
+
+    IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
+      result := result || w;
+    END IF;
+
+    words := string_to_array(s, ' ');
+    IF array_upper(words, 1) IS NOT NULL THEN
+      FOR j IN 1..array_upper(words, 1) LOOP
+        IF (words[j] != '') THEN
+          w = getorcreate_word_id(words[j]);
+          IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
+            result := result || w;
+          END IF;
+        END IF;
+      END LOOP;
+    END IF;
+
+    words := regexp_split_to_array(item.value, E'[,;()]');
+    IF array_upper(words, 1) != 1 THEN
+      FOR j IN 1..array_upper(words, 1) LOOP
+        s := make_standard_name(words[j]);
+        IF s != '' THEN
+          w := getorcreate_word_id(s);
+          IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
+            result := result || w;
+          END IF;
+        END IF;
+      END LOOP;
+    END IF;
+
+    s := regexp_replace(item.value, '市$', '');
+    IF s != item.value THEN
+      s := make_standard_name(s);
+      IF s != '' THEN
+        w := getorcreate_name_id(s, item.value);
+        IF NOT (ARRAY[w] <@ result) THEN
+          result := result || w;
+        END IF;
+      END IF;
+    END IF;
+
+  END LOOP;
+
+  RETURN result;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION make_keywords(src TEXT)
+  RETURNS INTEGER[]
+  AS $$
+DECLARE
+  result INTEGER[];
+  s TEXT;
+  w INTEGER;
+  words TEXT[];
+  i INTEGER;
+  j INTEGER;
+BEGIN
+  result := '{}'::INTEGER[];
+
+  s := make_standard_name(src);
+  w := getorcreate_name_id(s, src);
+
+  IF NOT (ARRAY[w] <@ result) THEN
+    result := result || w;
+  END IF;
+
+  w := getorcreate_word_id(s);
+
+  IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
+    result := result || w;
+  END IF;
+
+  words := string_to_array(s, ' ');
+  IF array_upper(words, 1) IS NOT NULL THEN
+    FOR j IN 1..array_upper(words, 1) LOOP
+      IF (words[j] != '') THEN
+        w = getorcreate_word_id(words[j]);
+        IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
+          result := result || w;
+        END IF;
+      END IF;
+    END LOOP;
+  END IF;
+
+  words := regexp_split_to_array(src, E'[,;()]');
+  IF array_upper(words, 1) != 1 THEN
+    FOR j IN 1..array_upper(words, 1) LOOP
+      s := make_standard_name(words[j]);
+      IF s != '' THEN
+        w := getorcreate_word_id(s);
+        IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
+          result := result || w;
+        END IF;
+      END IF;
+    END LOOP;
+  END IF;
+
+  s := regexp_replace(src, '市$', '');
+  IF s != src THEN
+    s := make_standard_name(s);
+    IF s != '' THEN
+      w := getorcreate_name_id(s, src);
+      IF NOT (ARRAY[w] <@ result) THEN
+        result := result || w;
+      END IF;
+    END IF;
+  END IF;
+
+  RETURN result;
+END;
+$$
+LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION create_poi_search_terms(obj_place_id BIGINT,
+                                                   in_partition SMALLINT,
+                                                   parent_place_id BIGINT,
+                                                   address HSTORE,
+                                                   country TEXT,
+                                                   housenumber TEXT,
+                                                   initial_name_vector INTEGER[],
+                                                   geometry GEOMETRY,
+                                                   OUT name_vector INTEGER[],
+                                                   OUT nameaddress_vector INTEGER[])
+  AS $$
+DECLARE
+  parent_name_vector INTEGER[];
+  parent_address_vector INTEGER[];
+  addr_place_ids INTEGER[];
+
+  addr_item RECORD;
+  parent_address_place_ids BIGINT[];
+  filtered_address HSTORE;
+BEGIN
+  nameaddress_vector := '{}'::INTEGER[];
+
+  SELECT s.name_vector, s.nameaddress_vector
+    INTO parent_name_vector, parent_address_vector
+    FROM search_name s
+    WHERE s.place_id = parent_place_id;
+
+  -- Find all address tags that don't appear in the parent search names.
+  SELECT hstore(array_agg(ARRAY[k, v])) INTO filtered_address
+    FROM (SELECT skeys(address) as k, svals(address) as v) a
+   WHERE not addr_ids_from_name(v) && parent_address_vector
+         AND k not in ('country', 'street', 'place', 'postcode',
+                       'housenumber', 'streetnumber', 'conscriptionnumber');
+
+  -- Compute all search terms from the addr: tags.
+  IF filtered_address IS NOT NULL THEN
+    FOR addr_item IN
+      SELECT * FROM
+        get_places_for_addr_tags(in_partition, geometry, filtered_address, country)
+    LOOP
+        IF addr_item.place_id is null THEN
+            nameaddress_vector := array_merge(nameaddress_vector,
+                                              addr_item.keywords);
+            CONTINUE;
+        END IF;
+
+        IF parent_address_place_ids is null THEN
+            SELECT array_agg(parent_place_id) INTO parent_address_place_ids
+              FROM place_addressline
+             WHERE place_id = parent_place_id;
+        END IF;
+
+        IF not parent_address_place_ids @> ARRAY[addr_item.place_id] THEN
+            nameaddress_vector := array_merge(nameaddress_vector,
+                                              addr_item.keywords);
+
+            INSERT INTO place_addressline (place_id, address_place_id, fromarea,
+                                           isaddress, distance, cached_rank_address)
+            VALUES (obj_place_id, addr_item.place_id, not addr_item.isguess,
+                    true, addr_item.distance, addr_item.rank_address);
+        END IF;
+    END LOOP;
+  END IF;
+
+  name_vector := initial_name_vector;
+
+  -- Check if the parent covers all address terms.
+  -- If not, create a search name entry with the house number as the name.
+  -- This is unusual for the search_name table but prevents that the place
+  -- is returned when we only search for the street/place.
+
+  IF housenumber is not null and not nameaddress_vector <@ parent_address_vector THEN
+    name_vector := array_merge(name_vector,
+                               ARRAY[getorcreate_housenumber_id(make_standard_name(housenumber))]);
+  END IF;
+
+  IF not address ? 'street' and address ? 'place' THEN
+    addr_place_ids := addr_ids_from_name(address->'place');
+    IF not addr_place_ids <@ parent_name_vector THEN
+      -- make sure addr:place terms are always searchable
+      nameaddress_vector := array_merge(nameaddress_vector, addr_place_ids);
+      -- If there is a housenumber, also add the place name as a name,
+      -- so we can search it by the usual housenumber+place algorithms.
+      IF housenumber is not null THEN
+        name_vector := array_merge(name_vector,
+                                   ARRAY[getorcreate_name_id(make_standard_name(address->'place'))]);
+      END IF;
+    END IF;
+  END IF;
+
+  -- Cheating here by not recomputing all terms but simply using the ones
+  -- from the parent object.
+  nameaddress_vector := array_merge(nameaddress_vector, parent_name_vector);
+  nameaddress_vector := array_merge(nameaddress_vector, parent_address_vector);
+
+END;
+$$
+LANGUAGE plpgsql;

lib-sql/functions/partition-functions.sql

@@ -63,36 +63,54 @@ END
 $$
 LANGUAGE plpgsql STABLE;
 
-
-CREATE OR REPLACE FUNCTION get_address_place(in_partition SMALLINT, feature GEOMETRY,
-                                             from_rank SMALLINT, to_rank SMALLINT,
-                                             extent FLOAT, tokens INT[])
-  RETURNS nearfeaturecentr
+CREATE OR REPLACE FUNCTION get_places_for_addr_tags(in_partition SMALLINT,
+                                                    feature GEOMETRY,
+                                                    address HSTORE, country TEXT)
+  RETURNS SETOF nearfeaturecentr
   AS $$
 DECLARE
   r nearfeaturecentr%rowtype;
+  item RECORD;
 BEGIN
+  FOR item IN
+    SELECT (get_addr_tag_rank(key, country)).*, key, name FROM
+      (SELECT skeys(address) as key, svals(address) as name) x
+  LOOP
+   IF item.from_rank is null THEN
+     CONTINUE;
+   END IF;
+
 {% for partition in db.partitions %}
-  IF in_partition = {{ partition }} THEN
-      SELECT place_id, keywords, rank_address, rank_search,
-             min(ST_Distance(feature, centroid)) as distance,
-             isguess, postcode, centroid INTO r
+    IF in_partition = {{ partition }} THEN
+        SELECT place_id, keywords, rank_address, rank_search,
+               min(ST_Distance(feature, centroid)) as distance,
+               isguess, postcode, centroid INTO r
         FROM location_area_large_{{ partition }}
-        WHERE geometry && ST_Expand(feature, extent)
-              AND rank_address between from_rank and to_rank
-              AND tokens && keywords
+        WHERE geometry && ST_Expand(feature, item.extent)
+          AND rank_address between item.from_rank and item.to_rank
+          AND word_ids_from_name(item.name) && keywords
         GROUP BY place_id, keywords, rank_address, rank_search, isguess, postcode, centroid
         ORDER BY bool_or(ST_Intersects(geometry, feature)), distance LIMIT 1;
-      RETURN r;
-  END IF;
+      IF r.place_id is null THEN
+        -- If we cannot find a place for the term, just return the
+        -- search term for the given name. That ensures that the address
+        -- element can still be searched for, even though it will not be
+        -- displayed.
+        RETURN NEXT ROW(null, addr_ids_from_name(item.name), null, null,
+                        null, null, null, null)::nearfeaturecentr;
+      ELSE
+        RETURN NEXT r;
+      END IF;
+      CONTINUE;
+    END IF;
 {% endfor %}
 
-  RAISE EXCEPTION 'Unknown partition %', in_partition;
+    RAISE EXCEPTION 'Unknown partition %', in_partition;
+  END LOOP;
 END;
 $$
 LANGUAGE plpgsql STABLE;
 
-
 create or replace function deleteLocationArea(in_partition INTEGER, in_place_id BIGINT, in_rank_search INTEGER) RETURNS BOOLEAN AS $$
 DECLARE
 BEGIN

lib-sql/functions/place_triggers.sql

@@ -99,27 +99,6 @@ BEGIN
       DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class;
     END IF;
 
-    -- Pure postcodes are never queried from placex so we don't add them.
-    -- location_postcodes is filled from the place table directly.
-    IF NEW.class = 'place' AND NEW.type = 'postcode' THEN
-      -- Remove old placex entry.
-      DELETE FROM placex where osm_type = NEW.osm_type and osm_id = NEW.osm_id;
-
-      IF existing.osm_type IS NOT NULL THEN
-        IF coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore)
-           OR existing.geometry::text != NEW.geometry::text
-        THEN
-
-          update place set address = NEW.address, geometry = NEW.geometry
-            where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
-        END IF;
-
-        RETURN NULL;
-      END IF;
-
-      RETURN NEW;
-    END IF;
-
     {% if debug %}RAISE WARNING 'Existing: %',existing.osm_id;{% endif %}
     {% if debug %}RAISE WARNING 'Existing PlaceX: %',existingplacex.place_id;{% endif %}
 
@@ -222,7 +201,7 @@ BEGIN
         where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
 
 
-      IF NEW.class = 'boundary' AND NEW.type = 'postal_code' THEN
+      IF NEW.class in ('place','boundary') AND NEW.type in ('postcode','postal_code') THEN
           IF NEW.address is NULL OR NOT NEW.address ? 'postcode' THEN
               -- postcode was deleted, no longer retain in placex
               DELETE FROM placex where place_id = existingplacex.place_id;

lib-sql/functions/placex_triggers.sql

@@ -1,84 +1,5 @@
 -- Trigger functions for the placex table.
 
--- Retrieve the data needed by the indexer for updating the place.
---
--- Return parameters:
---  name            list of names
---  address         list of address tags, either from the object or a surrounding
---                  building
---  country_feature If the place is a country feature, this contains the
---                  country code, otherwise it is null.
-CREATE OR REPLACE FUNCTION placex_prepare_update(p placex,
-                                                 OUT name HSTORE,
-                                                 OUT address HSTORE,
-                                                 OUT country_feature VARCHAR)
-  AS $$
-BEGIN
-  -- For POI nodes, check if the address should be derived from a surrounding
-  -- building.
-  IF p.rank_search < 30 OR p.osm_type != 'N' OR p.address is not null THEN
-    address := p.address;
-  ELSE
-    -- The additional && condition works around the misguided query
-    -- planner of postgis 3.0.
-    SELECT placex.address || hstore('_inherited', '') INTO address
-      FROM placex
-     WHERE ST_Covers(geometry, p.centroid)
-           and geometry && p.centroid
-           and placex.address is not null
-           and (placex.address ? 'housenumber' or placex.address ? 'street' or placex.address ? 'place')
-           and rank_search = 30 AND ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon')
-     LIMIT 1;
-  END IF;
-
-  address := address - '_unlisted_place'::TEXT;
-  name := p.name;
-
-  country_feature := CASE WHEN p.admin_level = 2
-                               and p.class = 'boundary' and p.type = 'administrative'
-                               and p.osm_type = 'R'
-                          THEN p.country_code
-                          ELSE null
-                     END;
-END;
-$$
-LANGUAGE plpgsql STABLE;
-
-
-CREATE OR REPLACE FUNCTION find_associated_street(poi_osm_type CHAR(1),
-                                                  poi_osm_id BIGINT)
-  RETURNS BIGINT
-  AS $$
-DECLARE
-  location RECORD;
-  parent RECORD;
-BEGIN
-  FOR location IN
-    SELECT members FROM planet_osm_rels
-    WHERE parts @> ARRAY[poi_osm_id]
-          and members @> ARRAY[lower(poi_osm_type) || poi_osm_id]
-          and tags @> ARRAY['associatedStreet']
-  LOOP
-    FOR i IN 1..array_upper(location.members, 1) BY 2 LOOP
-      IF location.members[i+1] = 'street' THEN
-        FOR parent IN
-          SELECT place_id from placex
-           WHERE osm_type = 'W' and osm_id = substring(location.members[i],2)::bigint
-                 and name is not null
-                 and rank_search between 26 and 27
-        LOOP
-          RETURN parent.place_id;
-        END LOOP;
-      END IF;
-    END LOOP;
-  END LOOP;
-
-  RETURN NULL;
-END;
-$$
-LANGUAGE plpgsql STABLE;
-
-
 -- Find the parent road of a POI.
 --
 -- \returns Place ID of parent object or NULL if none
@@ -89,89 +10,118 @@ CREATE OR REPLACE FUNCTION find_parent_for_poi(poi_osm_type CHAR(1),
                                                poi_osm_id BIGINT,
                                                poi_partition SMALLINT,
                                                bbox GEOMETRY,
-                                               addr_street INTEGER[],
-                                               addr_place INTEGER[],
-                                               is_place_addr BOOLEAN)
+                                               addr_street TEXT,
+                                               addr_place TEXT,
+                                               fallback BOOL = true)
   RETURNS BIGINT
   AS $$
 DECLARE
   parent_place_id BIGINT DEFAULT NULL;
   location RECORD;
+  parent RECORD;
 BEGIN
-  {% if debug %}RAISE WARNING 'finding street for % %', poi_osm_type, poi_osm_id;{% endif %}
+    {% if debug %}RAISE WARNING 'finding street for % %', poi_osm_type, poi_osm_id;{% endif %}
 
-  -- Is this object part of an associatedStreet relation?
-  parent_place_id := find_associated_street(poi_osm_type, poi_osm_id);
+    -- Is this object part of an associatedStreet relation?
+    FOR location IN
+      SELECT members FROM planet_osm_rels
+      WHERE parts @> ARRAY[poi_osm_id]
+        and members @> ARRAY[lower(poi_osm_type) || poi_osm_id]
+        and tags @> ARRAY['associatedStreet']
+    LOOP
+      FOR i IN 1..array_upper(location.members, 1) BY 2 LOOP
+        IF location.members[i+1] = 'street' THEN
+          FOR parent IN
+            SELECT place_id from placex
+             WHERE osm_type = 'W' and osm_id = substring(location.members[i],2)::bigint
+               and name is not null
+               and rank_search between 26 and 27
+          LOOP
+            RETURN parent.place_id;
+          END LOOP;
+        END IF;
+      END LOOP;
+    END LOOP;
 
-  IF parent_place_id is null THEN
     parent_place_id := find_parent_for_address(addr_street, addr_place,
                                                poi_partition, bbox);
-  END IF;
+    IF parent_place_id is not null THEN
+      RETURN parent_place_id;
+    END IF;
 
-  IF parent_place_id is null and poi_osm_type = 'N' THEN
-    -- Is this node part of an interpolation?
-    FOR location IN
-      SELECT q.parent_place_id
-        FROM location_property_osmline q, planet_osm_ways x
-       WHERE q.linegeo && bbox and x.id = q.osm_id
-             and poi_osm_id = any(x.nodes)
-       LIMIT 1
-    LOOP
-      {% if debug %}RAISE WARNING 'Get parent from interpolation: %', location.parent_place_id;{% endif %}
-      RETURN location.parent_place_id;
-    END LOOP;
-
-    FOR location IN
-      SELECT p.place_id, p.osm_id, p.rank_search, p.address,
-             coalesce(p.centroid, ST_Centroid(p.geometry)) as centroid
-        FROM placex p, planet_osm_ways w
-       WHERE p.osm_type = 'W' and p.rank_search >= 26
-             and p.geometry && bbox
-             and w.id = p.osm_id and poi_osm_id = any(w.nodes)
-    LOOP
-      {% if debug %}RAISE WARNING 'Node is part of way % ', location.osm_id;{% endif %}
-
-      -- Way IS a road then we are on it - that must be our road
-      IF location.rank_search < 28 THEN
-        {% if debug %}RAISE WARNING 'node in way that is a street %',location;{% endif %}
-        RETURN location.place_id;
-      END IF;
-
-      parent_place_id := find_associated_street('W', location.osm_id);
-    END LOOP;
-  END IF;
-
-  IF parent_place_id is NULL THEN
-    IF is_place_addr THEN
-      -- The address is attached to a place we don't know.
-      -- Instead simply use the containing area with the largest rank.
-      FOR location IN
-        SELECT place_id FROM placex
-         WHERE bbox && geometry AND _ST_Covers(geometry, ST_Centroid(bbox))
-               AND rank_address between 5 and 25
-         ORDER BY rank_address desc
+    IF poi_osm_type = 'N' THEN
+      -- Is this node part of an interpolation?
+      FOR parent IN
+        SELECT q.parent_place_id
+          FROM location_property_osmline q, planet_osm_ways x
+         WHERE q.linegeo && bbox and x.id = q.osm_id
+               and poi_osm_id = any(x.nodes)
+         LIMIT 1
       LOOP
-        RETURN location.place_id;
+        {% if debug %}RAISE WARNING 'Get parent from interpolation: %', parent.parent_place_id;{% endif %}
+        RETURN parent.parent_place_id;
       END LOOP;
-    ELSEIF ST_Area(bbox) < 0.005 THEN
-      -- for smaller features get the nearest road
-      SELECT getNearestRoadPlaceId(poi_partition, bbox) INTO parent_place_id;
-      {% if debug %}RAISE WARNING 'Checked for nearest way (%)', parent_place_id;{% endif %}
-    ELSE
-      -- for larger features simply find the area with the largest rank that
-      -- contains the bbox, only use addressable features
+
+      -- Is this node part of any other way?
       FOR location IN
-        SELECT place_id FROM placex
-         WHERE bbox && geometry AND _ST_Covers(geometry, ST_Centroid(bbox))
-               AND rank_address between 5 and 25
-        ORDER BY rank_address desc
+        SELECT p.place_id, p.osm_id, p.rank_search, p.address,
+               coalesce(p.centroid, ST_Centroid(p.geometry)) as centroid
+          FROM placex p, planet_osm_ways w
+         WHERE p.osm_type = 'W' and p.rank_search >= 26
+               and p.geometry && bbox
+               and w.id = p.osm_id and poi_osm_id = any(w.nodes)
       LOOP
-        RETURN location.place_id;
+        {% if debug %}RAISE WARNING 'Node is part of way % ', location.osm_id;{% endif %}
+
+        -- Way IS a road then we are on it - that must be our road
+        IF location.rank_search < 28 THEN
+          {% if debug %}RAISE WARNING 'node in way that is a street %',location;{% endif %}
+          return location.place_id;
+        END IF;
+
+        SELECT find_parent_for_poi('W', location.osm_id, poi_partition,
+                                   location.centroid,
+                                   location.address->'street',
+                                   location.address->'place',
+                                   false)
+          INTO parent_place_id;
+        IF parent_place_id is not null THEN
+          RETURN parent_place_id;
+        END IF;
       END LOOP;
     END IF;
-  END IF;
 
-  RETURN parent_place_id;
+    IF fallback THEN
+      IF addr_street is null and addr_place is not null THEN
+        -- The address is attached to a place we don't know.
+        -- Instead simply use the containing area with the largest rank.
+        FOR location IN
+          SELECT place_id FROM placex
+            WHERE bbox && geometry AND _ST_Covers(geometry, ST_Centroid(bbox))
+                  AND rank_address between 5 and 25
+            ORDER BY rank_address desc
+        LOOP
+            RETURN location.place_id;
+        END LOOP;
+      ELSEIF ST_Area(bbox) < 0.005 THEN
+        -- for smaller features get the nearest road
+        SELECT getNearestRoadPlaceId(poi_partition, bbox) INTO parent_place_id;
+        {% if debug %}RAISE WARNING 'Checked for nearest way (%)', parent_place_id;{% endif %}
+      ELSE
+        -- for larger features simply find the area with the largest rank that
+        -- contains the bbox, only use addressable features
+        FOR location IN
+          SELECT place_id FROM placex
+            WHERE bbox && geometry AND _ST_Covers(geometry, ST_Centroid(bbox))
+                  AND rank_address between 5 and 25
+            ORDER BY rank_address desc
+        LOOP
+            RETURN location.place_id;
+        END LOOP;
+      END IF;
+    END IF;
+
+    RETURN parent_place_id;
 END;
 $$
 LANGUAGE plpgsql STABLE;
@@ -219,7 +169,7 @@ BEGIN
   END IF;
 
   IF bnd.name ? 'name' THEN
-    bnd_name := lower(bnd.name->'name');
+    bnd_name := make_standard_name(bnd.name->'name');
     IF bnd_name = '' THEN
       bnd_name := NULL;
     END IF;
@@ -230,14 +180,12 @@ BEGIN
   IF bnd.extratags ? 'place' and bnd_name is not null THEN
     FOR linked_placex IN
       SELECT * FROM placex
-      WHERE (position(lower(name->'name') in bnd_name) > 0
-             OR position(bnd_name in lower(name->'name')) > 0)
+      WHERE make_standard_name(name->'name') = bnd_name
         AND placex.class = 'place' AND placex.type = bnd.extratags->'place'
         AND placex.osm_type = 'N'
         AND placex.linked_place_id is null
         AND placex.rank_search < 26 -- needed to select the right index
-        AND placex.type != 'postcode'
-        AND ST_Covers(bnd.geometry, placex.geometry)
+        AND _st_covers(bnd.geometry, placex.geometry)
     LOOP
       {% if debug %}RAISE WARNING 'Found type-matching place node %', linked_placex.osm_id;{% endif %}
       RETURN linked_placex;
@@ -253,7 +201,7 @@ BEGIN
         AND placex.linked_place_id is null
         AND placex.rank_search < 26
         AND _st_covers(bnd.geometry, placex.geometry)
-      ORDER BY lower(name->'name') = bnd_name desc
+      ORDER BY make_standard_name(name->'name') = bnd_name desc
     LOOP
       {% if debug %}RAISE WARNING 'Found wikidata-matching place node %', linked_placex.osm_id;{% endif %}
       RETURN linked_placex;
@@ -265,7 +213,7 @@ BEGIN
     {% if debug %}RAISE WARNING 'Looking for nodes with matching names';{% endif %}
     FOR linked_placex IN
       SELECT placex.* from placex
-      WHERE lower(name->'name') = bnd_name
+      WHERE make_standard_name(name->'name') = bnd_name
         AND ((bnd.rank_address > 0
               and bnd.rank_address = (compute_place_rank(placex.country_code,
                                                          'N', placex.class,
@@ -273,11 +221,9 @@ BEGIN
                                                          false, placex.postcode)).address_rank)
              OR (bnd.rank_address = 0 and placex.rank_search = bnd.rank_search))
         AND placex.osm_type = 'N'
-        AND placex.class = 'place'
         AND placex.linked_place_id is null
         AND placex.rank_search < 26 -- needed to select the right index
-        AND placex.type != 'postcode'
-        AND ST_Covers(bnd.geometry, placex.geometry)
+        AND _st_covers(bnd.geometry, placex.geometry)
     LOOP
       {% if debug %}RAISE WARNING 'Found matching place node %', linked_placex.osm_id;{% endif %}
       RETURN linked_placex;
@@ -290,101 +236,6 @@ $$
 LANGUAGE plpgsql STABLE;
 
 
-CREATE OR REPLACE FUNCTION create_poi_search_terms(obj_place_id BIGINT,
-                                                   in_partition SMALLINT,
-                                                   parent_place_id BIGINT,
-                                                   is_place_addr BOOLEAN,
-                                                   country TEXT,
-                                                   token_info JSONB,
-                                                   geometry GEOMETRY,
-                                                   OUT name_vector INTEGER[],
-                                                   OUT nameaddress_vector INTEGER[])
-  AS $$
-DECLARE
-  parent_name_vector INTEGER[];
-  parent_address_vector INTEGER[];
-  addr_place_ids INTEGER[];
-  hnr_vector INTEGER[];
-
-  addr_item RECORD;
-  addr_place RECORD;
-  parent_address_place_ids BIGINT[];
-BEGIN
-  nameaddress_vector := '{}'::INTEGER[];
-
-  SELECT s.name_vector, s.nameaddress_vector
-    INTO parent_name_vector, parent_address_vector
-    FROM search_name s
-    WHERE s.place_id = parent_place_id;
-
-  FOR addr_item IN
-    SELECT (get_addr_tag_rank(key, country)).*, match_tokens, search_tokens
-      FROM token_get_address_tokens(token_info)
-      WHERE not search_tokens <@ parent_address_vector
-  LOOP
-    addr_place := get_address_place(in_partition, geometry,
-                                    addr_item.from_rank, addr_item.to_rank,
-                                    addr_item.extent, addr_item.match_tokens);
-
-    IF addr_place is null THEN
-      -- No place found in OSM that matches. Make it at least searchable.
-      nameaddress_vector := array_merge(nameaddress_vector, addr_item.search_tokens);
-    ELSE
-      IF parent_address_place_ids is null THEN
-        SELECT array_agg(parent_place_id) INTO parent_address_place_ids
-          FROM place_addressline
-          WHERE place_id = parent_place_id;
-      END IF;
-
-      -- If the parent already lists the place in place_address line, then we
-      -- are done. Otherwise, add its own place_address line.
-      IF not parent_address_place_ids @> ARRAY[addr_place.place_id] THEN
-        nameaddress_vector := array_merge(nameaddress_vector, addr_place.keywords);
-
-        INSERT INTO place_addressline (place_id, address_place_id, fromarea,
-                                       isaddress, distance, cached_rank_address)
-          VALUES (obj_place_id, addr_place.place_id, not addr_place.isguess,
-                    true, addr_place.distance, addr_place.rank_address);
-      END IF;
-    END IF;
-  END LOOP;
-
-  name_vector := token_get_name_search_tokens(token_info);
-
-  -- Check if the parent covers all address terms.
-  -- If not, create a search name entry with the house number as the name.
-  -- This is unusual for the search_name table but prevents that the place
-  -- is returned when we only search for the street/place.
-
-  hnr_vector := token_get_housenumber_search_tokens(token_info);
-
-  IF hnr_vector is not null and not nameaddress_vector <@ parent_address_vector THEN
-    name_vector := array_merge(name_vector, hnr_vector);
-  END IF;
-
-  IF is_place_addr THEN
-    addr_place_ids := token_addr_place_search_tokens(token_info);
-    IF not addr_place_ids <@ parent_name_vector THEN
-      -- make sure addr:place terms are always searchable
-      nameaddress_vector := array_merge(nameaddress_vector, addr_place_ids);
-      -- If there is a housenumber, also add the place name as a name,
-      -- so we can search it by the usual housenumber+place algorithms.
-      IF hnr_vector is not null THEN
-        name_vector := array_merge(name_vector, addr_place_ids);
-      END IF;
-    END IF;
-  END IF;
-
-  -- Cheating here by not recomputing all terms but simply using the ones
-  -- from the parent object.
-  nameaddress_vector := array_merge(nameaddress_vector, parent_name_vector);
-  nameaddress_vector := array_merge(nameaddress_vector, parent_address_vector);
-
-END;
-$$
-LANGUAGE plpgsql;
-
-
 -- Insert address of a place into the place_addressline table.
 --
 -- \param obj_place_id  Place_id of the place to compute the address for.
@@ -405,7 +256,7 @@ LANGUAGE plpgsql;
 CREATE OR REPLACE FUNCTION insert_addresslines(obj_place_id BIGINT,
                                                partition SMALLINT,
                                                maxrank SMALLINT,
-                                               token_info JSONB,
+                                               address HSTORE,
                                                geometry GEOMETRY,
                                                country TEXT,
                                                OUT parent_place_id BIGINT,
@@ -420,8 +271,7 @@ DECLARE
   current_node_area GEOMETRY := NULL;
 
   parent_place_rank INT := 0;
-  addr_place_ids BIGINT[] := '{}'::int[];
-  new_address_vector INT[];
+  addr_place_ids BIGINT[];
 
   location RECORD;
 BEGIN
@@ -431,21 +281,16 @@ BEGIN
   address_havelevel := array_fill(false, ARRAY[maxrank]);
 
   FOR location IN
-    SELECT (get_address_place(partition, geometry, from_rank, to_rank,
-                              extent, match_tokens)).*, search_tokens
-      FROM (SELECT (get_addr_tag_rank(key, country)).*, match_tokens, search_tokens
-              FROM token_get_address_tokens(token_info)) x
-      ORDER BY rank_address, distance, isguess desc
+    SELECT * FROM get_places_for_addr_tags(partition, geometry,
+                                                   address, country)
+    ORDER BY rank_address, distance, isguess desc
   LOOP
-    IF location.place_id is null THEN
-      {% if not db.reverse_only %}
-      nameaddress_vector := array_merge(nameaddress_vector, location.search_tokens);
-      {% endif %}
-    ELSE
-      {% if not db.reverse_only %}
-      nameaddress_vector := array_merge(nameaddress_vector, location.keywords::INTEGER[]);
-      {% endif %}
+    {% if not db.reverse_only %}
+      nameaddress_vector := array_merge(nameaddress_vector,
+                                        location.keywords::int[]);
+    {% endif %}
 
+    IF location.place_id is not null THEN
       location_isaddress := not address_havelevel[location.rank_address];
       IF not address_havelevel[location.rank_address] THEN
         address_havelevel[location.rank_address] := true;
@@ -460,13 +305,13 @@ BEGIN
         VALUES (obj_place_id, location.place_id, not location.isguess,
                 true, location.distance, location.rank_address);
 
-      addr_place_ids := addr_place_ids || location.place_id;
+      addr_place_ids := array_append(addr_place_ids, location.place_id);
     END IF;
   END LOOP;
 
   FOR location IN
     SELECT * FROM getNearFeatures(partition, geometry, maxrank)
-    WHERE not addr_place_ids @> ARRAY[place_id]
+    WHERE addr_place_ids is null or not addr_place_ids @> ARRAY[place_id]
     ORDER BY rank_address, isguess asc,
              distance *
                CASE WHEN rank_address = 16 AND rank_search = 15 THEN 0.2
@@ -548,11 +393,10 @@ BEGIN
   NEW.place_id := nextval('seq_place');
   NEW.indexed_status := 1; --STATUS_NEW
 
-  NEW.centroid := ST_PointOnSurface(NEW.geometry);
-  NEW.country_code := lower(get_country_code(NEW.centroid));
+  NEW.country_code := lower(get_country_code(NEW.geometry));
 
   NEW.partition := get_partition(NEW.country_code);
-  NEW.geometry_sector := geometry_sector(NEW.partition, NEW.centroid);
+  NEW.geometry_sector := geometry_sector(NEW.partition, NEW.geometry);
 
   IF NEW.osm_type = 'X' THEN
     -- E'X'ternal records should already be in the right format so do nothing
@@ -674,8 +518,8 @@ DECLARE
   parent_address_level SMALLINT;
   place_address_level SMALLINT;
 
-  addr_street INTEGER[];
-  addr_place INTEGER[];
+  addr_street TEXT;
+  addr_place TEXT;
 
   max_rank SMALLINT;
 
@@ -683,11 +527,12 @@ DECLARE
   nameaddress_vector INTEGER[];
   addr_nameaddress_vector INTEGER[];
 
+  inherited_address HSTORE;
+
   linked_node_id BIGINT;
   linked_importance FLOAT;
   linked_wikipedia TEXT;
 
-  is_place_address BOOLEAN;
   result BOOLEAN;
 BEGIN
   -- deferred delete
@@ -717,9 +562,9 @@ BEGIN
   -- update not necessary for osmline, cause linked_place_id does not exist
 
   NEW.extratags := NEW.extratags - 'linked_place'::TEXT;
+  NEW.address := NEW.address - '_unlisted_place'::TEXT;
 
   IF NEW.linked_place_id is not null THEN
-    NEW.token_info := null;
     {% if debug %}RAISE WARNING 'place already linked to %', NEW.linked_place_id;{% endif %}
     RETURN NEW;
   END IF;
@@ -730,34 +575,13 @@ BEGIN
   -- imported as place=postcode. That's why relations are allowed to pass here.
   -- This can go away in a couple of versions.
   IF NEW.class = 'place'  and NEW.type = 'postcode' and NEW.osm_type != 'R' THEN
-    NEW.token_info := null;
     RETURN NEW;
   END IF;
 
-  -- Compute a preliminary centroid.
+  -- Speed up searches - just use the centroid of the feature
+  -- cheaper but less acurate
   NEW.centroid := ST_PointOnSurface(NEW.geometry);
-
-    -- recalculate country and partition
-  IF NEW.rank_search = 4 AND NEW.address is not NULL AND NEW.address ? 'country' THEN
-    -- for countries, believe the mapped country code,
-    -- so that we remain in the right partition if the boundaries
-    -- suddenly expand.
-    NEW.country_code := lower(NEW.address->'country');
-    NEW.partition := get_partition(lower(NEW.country_code));
-    IF NEW.partition = 0 THEN
-      NEW.country_code := lower(get_country_code(NEW.centroid));
-      NEW.partition := get_partition(NEW.country_code);
-    END IF;
-  ELSE
-    IF NEW.rank_search >= 4 THEN
-      NEW.country_code := lower(get_country_code(NEW.centroid));
-    ELSE
-      NEW.country_code := NULL;
-    END IF;
-    NEW.partition := get_partition(NEW.country_code);
-  END IF;
-  {% if debug %}RAISE WARNING 'Country updated: "%"', NEW.country_code;{% endif %}
-
+  {% if debug %}RAISE WARNING 'Computing preliminary centroid at %',ST_AsText(NEW.centroid);{% endif %}
 
   -- recompute the ranks, they might change when linking changes
   SELECT * INTO NEW.rank_search, NEW.rank_address
@@ -837,12 +661,54 @@ BEGIN
     parent_address_level := 3;
   END IF;
 
-  NEW.housenumber := token_normalized_housenumber(NEW.token_info);
-  addr_street := token_addr_street_match_tokens(NEW.token_info);
-  addr_place := token_addr_place_match_tokens(NEW.token_info);
+  {% if debug %}RAISE WARNING 'Copy over address tags';{% endif %}
+  -- housenumber is a computed field, so start with an empty value
+  NEW.housenumber := NULL;
+  IF NEW.address is not NULL THEN
+      IF NEW.address ? 'conscriptionnumber' THEN
+        IF NEW.address ? 'streetnumber' THEN
+            NEW.housenumber := (NEW.address->'conscriptionnumber') || '/' || (NEW.address->'streetnumber');
+        ELSE
+            NEW.housenumber := NEW.address->'conscriptionnumber';
+        END IF;
+      ELSEIF NEW.address ? 'streetnumber' THEN
+        NEW.housenumber := NEW.address->'streetnumber';
+      ELSEIF NEW.address ? 'housenumber' THEN
+        NEW.housenumber := NEW.address->'housenumber';
+      END IF;
+      NEW.housenumber := create_housenumber_id(NEW.housenumber);
+
+      addr_street := NEW.address->'street';
+      addr_place := NEW.address->'place';
+
+      IF NEW.address ? 'postcode' and NEW.address->'postcode' not similar to '%(:|,|;)%' THEN
+        i := getorcreate_postcode_id(NEW.address->'postcode');
+      END IF;
+  END IF;
 
   NEW.postcode := null;
 
+  -- recalculate country and partition
+  IF NEW.rank_search = 4 AND NEW.address is not NULL AND NEW.address ? 'country' THEN
+    -- for countries, believe the mapped country code,
+    -- so that we remain in the right partition if the boundaries
+    -- suddenly expand.
+    NEW.country_code := lower(NEW.address->'country');
+    NEW.partition := get_partition(lower(NEW.country_code));
+    IF NEW.partition = 0 THEN
+      NEW.country_code := lower(get_country_code(NEW.centroid));
+      NEW.partition := get_partition(NEW.country_code);
+    END IF;
+  ELSE
+    IF NEW.rank_search >= 4 THEN
+      NEW.country_code := lower(get_country_code(NEW.centroid));
+    ELSE
+      NEW.country_code := NULL;
+    END IF;
+    NEW.partition := get_partition(NEW.country_code);
+  END IF;
+  {% if debug %}RAISE WARNING 'Country updated: "%"', NEW.country_code;{% endif %}
+
   -- waterway ways are linked when they are part of a relation and have the same class/type
   IF NEW.osm_type = 'R' and NEW.class = 'waterway' THEN
       FOR relation_members IN select members from planet_osm_rels r where r.id = NEW.osm_id and r.parts != array[]::bigint[]
@@ -879,14 +745,33 @@ BEGIN
 
     {% if debug %}RAISE WARNING 'finding street for % %', NEW.osm_type, NEW.osm_id;{% endif %}
     NEW.parent_place_id := null;
-    is_place_address := coalesce(not NEW.address ? 'street' and NEW.address ? 'place', FALSE);
+
+    -- if we have a POI and there is no address information,
+    -- see if we can get it from a surrounding building
+    inherited_address := ''::HSTORE;
+    IF NEW.osm_type = 'N' AND addr_street IS NULL AND addr_place IS NULL
+       AND NEW.housenumber IS NULL THEN
+      FOR location IN
+        -- The additional && condition works around the misguided query
+        -- planner of postgis 3.0.
+        SELECT address from placex where ST_Covers(geometry, NEW.centroid)
+            and geometry && NEW.centroid
+            and (address ? 'housenumber' or address ? 'street' or address ? 'place')
+            and rank_search > 28 AND ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon')
+            limit 1
+      LOOP
+        NEW.housenumber := location.address->'housenumber';
+        addr_street := location.address->'street';
+        addr_place := location.address->'place';
+        inherited_address := location.address;
+      END LOOP;
+    END IF;
 
     -- We have to find our parent road.
     NEW.parent_place_id := find_parent_for_poi(NEW.osm_type, NEW.osm_id,
                                                NEW.partition,
                                                ST_Envelope(NEW.geometry),
-                                               addr_street, addr_place,
-                                               is_place_address);
+                                               addr_street, addr_place);
 
     -- If we found the road take a shortcut here.
     -- Otherwise fall back to the full address getting method below.
@@ -896,12 +781,12 @@ BEGIN
       SELECT p.country_code, p.postcode, p.name FROM placex p
        WHERE p.place_id = NEW.parent_place_id INTO location;
 
-      IF is_place_address THEN
+      IF addr_street is null and addr_place is not null THEN
         -- Check if the addr:place tag is part of the parent name
         SELECT count(*) INTO i
-          FROM svals(location.name) AS pname WHERE pname = NEW.address->'place';
+          FROM svals(location.name) AS pname WHERE pname = addr_place;
         IF i = 0 THEN
-          NEW.address = NEW.address || hstore('_unlisted_place', NEW.address->'place');
+          NEW.address = NEW.address || hstore('_unlisted_place', addr_place);
         END IF;
       END IF;
 
@@ -909,21 +794,39 @@ BEGIN
       {% if debug %}RAISE WARNING 'Got parent details from search name';{% endif %}
 
       -- determine postcode
-      NEW.postcode := coalesce(token_normalized_postcode(NEW.address->'postcode'),
-                               location.postcode,
-                               get_nearest_postcode(NEW.country_code, NEW.centroid));
+      IF NEW.address is not null AND NEW.address ? 'postcode' THEN
+          NEW.postcode = upper(trim(NEW.address->'postcode'));
+      ELSE
+         NEW.postcode := location.postcode;
+      END IF;
+      IF NEW.postcode is null THEN
+        NEW.postcode := get_nearest_postcode(NEW.country_code, NEW.geometry);
+      END IF;
 
       IF NEW.name is not NULL THEN
           NEW.name := add_default_place_name(NEW.country_code, NEW.name);
+          name_vector := make_keywords(NEW.name);
+
+          IF NEW.rank_search <= 25 and NEW.rank_address > 0 THEN
+            result := add_location(NEW.place_id, NEW.country_code, NEW.partition,
+                                   name_vector, NEW.rank_search, NEW.rank_address,
+                                   upper(trim(NEW.address->'postcode')), NEW.geometry,
+                                   NEW.centroid);
+            {% if debug %}RAISE WARNING 'Place added to location table';{% endif %}
+          END IF;
+
       END IF;
 
       {% if not db.reverse_only %}
-      IF NEW.name is not NULL OR NEW.address is not NULL THEN
+      IF array_length(name_vector, 1) is not NULL
+         OR inherited_address is not NULL OR NEW.address is not NULL
+      THEN
         SELECT * INTO name_vector, nameaddress_vector
           FROM create_poi_search_terms(NEW.place_id,
                                        NEW.partition, NEW.parent_place_id,
-                                       is_place_address, NEW.country_code,
-                                       NEW.token_info, NEW.centroid);
+                                       inherited_address || NEW.address,
+                                       NEW.country_code, NEW.housenumber,
+                                       name_vector, NEW.centroid);
 
         IF array_length(name_vector, 1) is not NULL THEN
           INSERT INTO search_name (place_id, search_rank, address_rank,
@@ -937,17 +840,6 @@ BEGIN
       END IF;
       {% endif %}
 
-      NEW.token_info := token_strip_info(NEW.token_info);
-      -- If the address was inherited from a surrounding building,
-      -- do not add it permanently to the table.
-      IF NEW.address ? '_inherited' THEN
-        IF NEW.address ? '_unlisted_place' THEN
-          NEW.address := hstore('_unlisted_place', NEW.address->'_unlisted_place');
-        ELSE
-          NEW.address := null;
-        END IF;
-      END IF;
-
       RETURN NEW;
     END IF;
 
@@ -1018,11 +910,19 @@ BEGIN
     END IF;
   END IF;
 
+  -- Initialise the name vector using our name
+  NEW.name := add_default_place_name(NEW.country_code, NEW.name);
+  name_vector := make_keywords(NEW.name);
+
+  -- make sure all names are in the word table
   IF NEW.admin_level = 2
      AND NEW.class = 'boundary' AND NEW.type = 'administrative'
      AND NEW.country_code IS NOT NULL AND NEW.osm_type = 'R'
   THEN
-    -- Update the list of country names. Adding an additional sanity
+    PERFORM create_country(NEW.name, lower(NEW.country_code));
+    {% if debug %}RAISE WARNING 'Country names updated';{% endif %}
+
+    -- Also update the list of country names. Adding an additional sanity
     -- check here: make sure the country does overlap with the area where
     -- we expect it to be as per static country grid.
     FOR location IN
@@ -1055,28 +955,29 @@ BEGIN
   ELSEIF NEW.rank_address > 25 THEN
     max_rank := 25;
   ELSE
-    max_rank := NEW.rank_address;
+    max_rank = NEW.rank_address;
   END IF;
 
   SELECT * FROM insert_addresslines(NEW.place_id, NEW.partition, max_rank,
-                                    NEW.token_info, geom, NEW.country_code)
+                                    NEW.address, geom, NEW.country_code)
     INTO NEW.parent_place_id, NEW.postcode, nameaddress_vector;
 
   {% if debug %}RAISE WARNING 'RETURN insert_addresslines: %, %, %', NEW.parent_place_id, NEW.postcode, nameaddress_vector;{% endif %}
 
-  NEW.postcode := coalesce(token_normalized_postcode(NEW.address->'postcode'),
-                           NEW.postcode);
+  IF NEW.address is not null AND NEW.address ? 'postcode' 
+     AND NEW.address->'postcode' not similar to '%(,|;)%' THEN
+    NEW.postcode := upper(trim(NEW.address->'postcode'));
+  END IF;
+
+  IF NEW.postcode is null AND NEW.rank_search > 8 THEN
+    NEW.postcode := get_nearest_postcode(NEW.country_code, NEW.geometry);
+  END IF;
 
   -- if we have a name add this to the name search table
   IF NEW.name IS NOT NULL THEN
-    -- Initialise the name vector using our name
-    NEW.name := add_default_place_name(NEW.country_code, NEW.name);
-    name_vector := token_get_name_search_tokens(NEW.token_info);
 
     IF NEW.rank_search <= 25 and NEW.rank_address > 0 THEN
-      result := add_location(NEW.place_id, NEW.country_code, NEW.partition,
-                             name_vector, NEW.rank_search, NEW.rank_address,
-                             NEW.postcode, NEW.geometry, NEW.centroid);
+      result := add_location(NEW.place_id, NEW.country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, upper(trim(NEW.address->'postcode')), NEW.geometry, NEW.centroid);
       {% if debug %}RAISE WARNING 'added to location (full)';{% endif %}
     END IF;
 
@@ -1085,11 +986,8 @@ BEGIN
       {% if debug %}RAISE WARNING 'insert into road location table (full)';{% endif %}
     END IF;
 
-    IF NEW.rank_address between 16 and 27 THEN
-      result := insertSearchName(NEW.partition, NEW.place_id,
-                                 token_get_name_match_tokens(NEW.token_info),
-                                 NEW.rank_search, NEW.rank_address, NEW.geometry);
-    END IF;
+    result := insertSearchName(NEW.partition, NEW.place_id, name_vector,
+                               NEW.rank_search, NEW.rank_address, NEW.geometry);
     {% if debug %}RAISE WARNING 'added to search name (full)';{% endif %}
 
     {% if not db.reverse_only %}
@@ -1100,15 +998,11 @@ BEGIN
                        NEW.importance, NEW.country_code, name_vector,
                        nameaddress_vector, NEW.centroid);
     {% endif %}
-  END IF;
 
-  IF NEW.postcode is null AND NEW.rank_search > 8 THEN
-    NEW.postcode := get_nearest_postcode(NEW.country_code, NEW.geometry);
   END IF;
 
   {% if debug %}RAISE WARNING 'place update % % finsihed.', NEW.osm_type, NEW.osm_id;{% endif %}
 
-  NEW.token_info := token_strip_info(NEW.token_info);
   RETURN NEW;
 END;
 $$

lib-sql/functions/utils.sql

@@ -221,30 +221,37 @@ LANGUAGE plpgsql STABLE;
 -- \param centroid   Location of the address.
 --
 -- \return Place ID of the parent if one was found, NULL otherwise.
-CREATE OR REPLACE FUNCTION find_parent_for_address(street INTEGER[], place INTEGER[],
+CREATE OR REPLACE FUNCTION find_parent_for_address(street TEXT, place TEXT,
                                                    partition SMALLINT,
                                                    centroid GEOMETRY)
   RETURNS BIGINT
   AS $$
 DECLARE
   parent_place_id BIGINT;
+  word_ids INTEGER[];
 BEGIN
   IF street is not null THEN
     -- Check for addr:street attributes
     -- Note that addr:street links can only be indexed, once the street itself is indexed
-    parent_place_id := getNearestNamedRoadPlaceId(partition, centroid, street);
-    IF parent_place_id is not null THEN
-      {% if debug %}RAISE WARNING 'Get parent form addr:street: %', parent_place_id;{% endif %}
-      RETURN parent_place_id;
+    word_ids := word_ids_from_name(street);
+    IF word_ids is not null THEN
+      parent_place_id := getNearestNamedRoadPlaceId(partition, centroid, word_ids);
+      IF parent_place_id is not null THEN
+        {% if debug %}RAISE WARNING 'Get parent form addr:street: %', parent_place_id;{% endif %}
+        RETURN parent_place_id;
+      END IF;
     END IF;
   END IF;
 
   -- Check for addr:place attributes.
   IF place is not null THEN
-    parent_place_id := getNearestNamedPlacePlaceId(partition, centroid, place);
-    IF parent_place_id is not null THEN
-      {% if debug %}RAISE WARNING 'Get parent form addr:place: %', parent_place_id;{% endif %}
-      RETURN parent_place_id;
+    word_ids := word_ids_from_name(place);
+    IF word_ids is not null THEN
+      parent_place_id := getNearestNamedPlacePlaceId(partition, centroid, word_ids);
+      IF parent_place_id is not null THEN
+        {% if debug %}RAISE WARNING 'Get parent form addr:place: %', parent_place_id;{% endif %}
+        RETURN parent_place_id;
+      END IF;
     END IF;
   END IF;
 

lib-sql/indices.sql

@@ -1,62 +1,71 @@
 -- Indices used only during search and update.
 -- These indices are created only after the indexing process is done.
 
-CREATE INDEX IF NOT EXISTS idx_place_addressline_address_place_id
+CREATE INDEX {{sql.if_index_not_exists}} idx_word_word_id
+  ON word USING BTREE (word_id) {{db.tablespace.search_index}};
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_place_addressline_address_place_id
   ON place_addressline USING BTREE (address_place_id) {{db.tablespace.search_index}};
 
-CREATE INDEX IF NOT EXISTS idx_placex_rank_search
+CREATE INDEX {{sql.if_index_not_exists}} idx_placex_rank_search
   ON placex USING BTREE (rank_search) {{db.tablespace.search_index}};
 
-CREATE INDEX IF NOT EXISTS idx_placex_rank_address
+CREATE INDEX {{sql.if_index_not_exists}} idx_placex_rank_address
   ON placex USING BTREE (rank_address) {{db.tablespace.search_index}};
 
-CREATE INDEX IF NOT EXISTS idx_placex_parent_place_id
+CREATE INDEX {{sql.if_index_not_exists}} idx_placex_parent_place_id
   ON placex USING BTREE (parent_place_id) {{db.tablespace.search_index}}
   WHERE parent_place_id IS NOT NULL;
 
-CREATE INDEX IF NOT EXISTS idx_placex_geometry_reverse_lookupPolygon
+CREATE INDEX {{sql.if_index_not_exists}} idx_placex_geometry_reverse_lookupPolygon
   ON placex USING gist (geometry) {{db.tablespace.search_index}}
   WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')
     AND rank_address between 4 and 25 AND type != 'postcode'
     AND name is not null AND indexed_status = 0 AND linked_place_id is null;
 
-CREATE INDEX IF NOT EXISTS idx_osmline_parent_place_id
+CREATE INDEX {{sql.if_index_not_exists}} idx_placex_geometry_reverse_placeNode
+  ON placex USING gist (geometry) {{db.tablespace.search_index}}
+  WHERE osm_type = 'N' AND rank_search between 5 and 25
+    AND class = 'place' AND type != 'postcode'
+    AND name is not null AND indexed_status = 0 AND linked_place_id is null;
+
+CREATE INDEX {{sql.if_index_not_exists}} idx_osmline_parent_place_id
   ON location_property_osmline USING BTREE (parent_place_id) {{db.tablespace.search_index}};
 
-CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id
+CREATE INDEX {{sql.if_index_not_exists}} idx_osmline_parent_osm_id
   ON location_property_osmline USING BTREE (osm_id) {{db.tablespace.search_index}};
 
-CREATE INDEX IF NOT EXISTS idx_postcode_postcode
+CREATE INDEX {{sql.if_index_not_exists}} idx_postcode_postcode
   ON location_postcode USING BTREE (postcode) {{db.tablespace.search_index}};
 
 -- Indices only needed for updating.
 
 {% if not drop %}
-  CREATE INDEX IF NOT EXISTS idx_placex_pendingsector
+  CREATE INDEX {{sql.if_index_not_exists}} idx_placex_pendingsector
     ON placex USING BTREE (rank_address,geometry_sector) {{db.tablespace.address_index}}
     WHERE indexed_status > 0;
 
-  CREATE INDEX IF NOT EXISTS idx_location_area_country_place_id
+  CREATE INDEX {{sql.if_index_not_exists}} idx_location_area_country_place_id
     ON location_area_country USING BTREE (place_id) {{db.tablespace.address_index}};
 
-  CREATE UNIQUE INDEX IF NOT EXISTS idx_place_osm_unique
+  CREATE UNIQUE INDEX {{sql.if_index_not_exists}} idx_place_osm_unique
     ON place USING btree(osm_id, osm_type, class, type) {{db.tablespace.address_index}};
 {% endif %}
 
 -- Indices only needed for search.
 
 {% if 'search_name' in db.tables %}
-  CREATE INDEX IF NOT EXISTS idx_search_name_nameaddress_vector
+  CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_nameaddress_vector
     ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
-  CREATE INDEX IF NOT EXISTS idx_search_name_name_vector
+  CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_name_vector
     ON search_name USING GIN (name_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
-  CREATE INDEX IF NOT EXISTS idx_search_name_centroid
+  CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_centroid
     ON search_name USING GIST (centroid) {{db.tablespace.search_index}};
 
   {% if postgres.has_index_non_key_column %}
-    CREATE INDEX IF NOT EXISTS idx_placex_housenumber
+    CREATE INDEX {{sql.if_index_not_exists}} idx_placex_housenumber
       ON placex USING btree (parent_place_id) INCLUDE (housenumber) WHERE housenumber is not null;
-    CREATE INDEX IF NOT EXISTS idx_osmline_parent_osm_id_with_hnr
+    CREATE INDEX {{sql.if_index_not_exists}} idx_osmline_parent_osm_id_with_hnr
       ON location_property_osmline USING btree(parent_place_id) INCLUDE (startnumber, endnumber);
   {% endif %}
 {% endif %}

lib-sql/tables.sql

@@ -43,6 +43,22 @@ CREATE TABLE nominatim_properties (
 );
 GRANT SELECT ON TABLE nominatim_properties TO "{{config.DATABASE_WEBUSER}}";
 
+drop table IF EXISTS word;
+CREATE TABLE word (
+  word_id INTEGER,
+  word_token text,
+  word text,
+  class text,
+  type text,
+  country_code varchar(2),
+  search_name_count INTEGER,
+  operator TEXT
+  ) {{db.tablespace.search_data}};
+CREATE INDEX idx_word_word_token on word USING BTREE (word_token) {{db.tablespace.search_index}};
+GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}" ;
+DROP SEQUENCE IF EXISTS seq_word;
+CREATE SEQUENCE seq_word start 1;
+
 drop table IF EXISTS location_area CASCADE;
 CREATE TABLE location_area (
   place_id BIGINT,
@@ -68,6 +84,22 @@ CREATE TABLE location_area_country (
 CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {{db.tablespace.address_index}};
 
 
+drop table IF EXISTS location_property CASCADE;
+CREATE TABLE location_property (
+  place_id BIGINT,
+  parent_place_id BIGINT,
+  partition SMALLINT,
+  housenumber TEXT,
+  postcode TEXT,
+  centroid GEOMETRY(Point, 4326)
+  );
+
+CREATE TABLE location_property_aux () INHERITS (location_property);
+CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
+CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
+CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
+GRANT SELECT ON location_property_aux TO "{{config.DATABASE_WEBUSER}}";
+
 CREATE TABLE location_property_tiger (
   place_id BIGINT,
   parent_place_id BIGINT,
@@ -93,7 +125,6 @@ CREATE TABLE location_property_osmline (
     linegeo GEOMETRY,
     interpolationtype TEXT,
     address HSTORE,
-    token_info JSONB, -- custom column for tokenizer use only
     postcode TEXT,
     country_code VARCHAR(2)
   ){{db.tablespace.search_data}};
@@ -143,7 +174,6 @@ CREATE TABLE placex (
   indexed_status SMALLINT,
   LIKE place INCLUDING CONSTRAINTS,
   wikipedia TEXT, -- calculated wikipedia article name (language:title)
-  token_info JSONB, -- custom column for tokenizer use only
   country_code varchar(2),
   housenumber TEXT,
   postcode TEXT,
@@ -154,20 +184,14 @@ CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {{db.tabl
 CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {{db.tablespace.address_index}} WHERE linked_place_id IS NOT NULL;
 CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}};
 CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {{db.tablespace.search_index}};
-CREATE INDEX idx_placex_geometry_buildings ON placex
-  USING GIST (geometry) {{db.tablespace.search_index}}
-  WHERE address is not null and rank_search = 30
-        and ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon');
-CREATE INDEX idx_placex_geometry_placenode ON placex
-  USING GIST (geometry) {{db.tablespace.search_index}}
-  WHERE osm_type = 'N' and rank_search < 26
-        and class = 'place' and type != 'postcode' and linked_place_id is null;
+CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name->'name')) {{db.tablespace.address_index}} WHERE osm_type='N' and rank_search < 26;
 CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26;
 
 DROP SEQUENCE IF EXISTS seq_place;
 CREATE SEQUENCE seq_place start 1;
 GRANT SELECT on placex to "{{config.DATABASE_WEBUSER}}" ;
 GRANT SELECT on place_addressline to "{{config.DATABASE_WEBUSER}}" ;
+GRANT SELECT ON seq_word to "{{config.DATABASE_WEBUSER}}" ;
 GRANT SELECT ON planet_osm_ways to "{{config.DATABASE_WEBUSER}}" ;
 GRANT SELECT ON planet_osm_rels to "{{config.DATABASE_WEBUSER}}" ;
 GRANT SELECT on location_area to "{{config.DATABASE_WEBUSER}}" ;

lib-sql/tiger_import_finish.sql

@@ -1,7 +1,7 @@
 --index only on parent_place_id
-CREATE INDEX IF NOT EXISTS idx_location_property_tiger_parent_place_id_imp
+CREATE INDEX {{sql.if_index_not_exists}} idx_location_property_tiger_parent_place_id_imp
   ON location_property_tiger_import (parent_place_id) {{db.tablespace.aux_index}};
-CREATE UNIQUE INDEX IF NOT EXISTS idx_location_property_tiger_place_id_imp
+CREATE UNIQUE INDEX {{sql.if_index_not_exists}} idx_location_property_tiger_place_id_imp
   ON location_property_tiger_import (place_id) {{db.tablespace.aux_index}};
 
 GRANT SELECT ON location_property_tiger_import TO "{{config.DATABASE_WEBUSER}}";
@@ -12,6 +12,4 @@ ALTER TABLE location_property_tiger_import RENAME TO location_property_tiger;
 ALTER INDEX IF EXISTS idx_location_property_tiger_parent_place_id_imp RENAME TO idx_location_property_tiger_housenumber_parent_place_id;
 ALTER INDEX IF EXISTS idx_location_property_tiger_place_id_imp RENAME TO idx_location_property_tiger_place_id;
 
-DROP FUNCTION tiger_line_import (linegeo GEOMETRY, in_startnumber INTEGER,
-                                 in_endnumber INTEGER, interpolationtype TEXT,
-                                 token_info JSONB, in_postcode TEXT);
+DROP FUNCTION tiger_line_import (linegeo geometry, in_startnumber integer, in_endnumber integer, interpolationtype text, in_street text, in_isin text, in_postcode text);

lib-sql/tiger_import_start.sql

@@ -1,9 +1,9 @@
 DROP TABLE IF EXISTS location_property_tiger_import;
 CREATE TABLE location_property_tiger_import (linegeo GEOMETRY, place_id BIGINT, partition INTEGER, parent_place_id BIGINT, startnumber INTEGER, endnumber INTEGER, interpolationtype TEXT, postcode TEXT);
 
-CREATE OR REPLACE FUNCTION tiger_line_import(linegeo GEOMETRY, in_startnumber INTEGER,
-                                             in_endnumber INTEGER, interpolationtype TEXT,
-                                             token_info JSONB, in_postcode TEXT) RETURNS INTEGER
+CREATE OR REPLACE FUNCTION tiger_line_import(linegeo GEOMETRY, in_startnumber INTEGER, 
+  in_endnumber INTEGER, interpolationtype TEXT,
+  in_street TEXT, in_isin TEXT, in_postcode TEXT) RETURNS INTEGER
   AS $$
 DECLARE
   startnumber INTEGER;
@@ -27,13 +27,13 @@ BEGIN
   END IF;
 
   IF startnumber < 0 THEN
-    RAISE WARNING 'Negative house number range (% to %)', startnumber, endnumber;
+    RAISE WARNING 'Negative house number range (% to %) on %, %', startnumber, endnumber, in_street, in_isin;
     RETURN 0;
   END IF;
 
   numberrange := endnumber - startnumber;
 
-  IF (interpolationtype = 'odd' AND startnumber % 2 = 0) OR (interpolationtype = 'even' AND startnumber % 2 = 1) THEN
+  IF (interpolationtype = 'odd' AND startnumber%2 = 0) OR (interpolationtype = 'even' AND startnumber%2 = 1) THEN
     startnumber := startnumber + 1;
     stepsize := 2;
   ELSE
@@ -45,10 +45,10 @@ BEGIN
   END IF;
 
   -- Filter out really broken tiger data
-  IF numberrange > 0 AND (numberrange::float/stepsize::float > 500)
+  IF numberrange > 0 AND (numberrange::float/stepsize::float > 500) 
                     AND ST_length(linegeo)/(numberrange::float/stepsize::float) < 0.000001 THEN
-    RAISE WARNING 'Road too short for number range % to % (%)',startnumber,endnumber,
-                  ST_length(linegeo)/(numberrange::float/stepsize::float);
+    RAISE WARNING 'Road too short for number range % to % on %, % (%)',startnumber,endnumber,in_street,in_isin,
+                  ST_length(linegeo)/(numberrange::float/stepsize::float);    
     RETURN 0;
   END IF;
 
@@ -56,7 +56,7 @@ BEGIN
   out_partition := get_partition('us');
   out_parent_place_id := null;
 
-  address_street_word_ids := token_addr_street_match_tokens(token_info);
+  address_street_word_ids := word_ids_from_name(in_street);
   IF address_street_word_ids IS NOT NULL THEN
     out_parent_place_id := getNearestNamedRoadPlaceId(out_partition, place_centroid,
                                                       address_street_word_ids);

lib-sql/tokenizer/icu_tokenizer_tables.sql

@@ -1,29 +0,0 @@
-DROP TABLE IF EXISTS word;
-CREATE TABLE word (
-  word_id INTEGER,
-  word_token text NOT NULL,
-  type text NOT NULL,
-  word text,
-  info jsonb
-) {{db.tablespace.search_data}};
-
-CREATE INDEX idx_word_word_token ON word
-    USING BTREE (word_token) {{db.tablespace.search_index}};
--- Used when updating country names from the boundary relation.
-CREATE INDEX idx_word_country_names ON word
-    USING btree(word) {{db.tablespace.address_index}}
-    WHERE type = 'C';
--- Used when inserting new postcodes on updates.
-CREATE INDEX idx_word_postcodes ON word
-    USING btree(word) {{db.tablespace.address_index}}
-    WHERE type = 'P';
--- Used when inserting full words.
-CREATE INDEX idx_word_full_word ON word
-    USING btree(word) {{db.tablespace.address_index}}
-    WHERE type = 'W';
-
-GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}";
-
-DROP SEQUENCE IF EXISTS seq_word;
-CREATE SEQUENCE seq_word start 1;
-GRANT SELECT ON seq_word to "{{config.DATABASE_WEBUSER}}";

lib-sql/tokenizer/legacy_icu_tokenizer.sql

@@ -1,157 +0,0 @@
--- Get tokens used for searching the given place.
---
--- These are the tokens that will be saved in the search_name table.
-CREATE OR REPLACE FUNCTION token_get_name_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'names')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Get tokens for matching the place name against others.
---
--- This should usually be restricted to full name tokens.
-CREATE OR REPLACE FUNCTION token_get_name_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'names')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return the housenumber tokens applicable for the place.
-CREATE OR REPLACE FUNCTION token_get_housenumber_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'hnr_tokens')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return the housenumber in the form that it can be matched during search.
-CREATE OR REPLACE FUNCTION token_normalized_housenumber(info JSONB)
-  RETURNS TEXT
-AS $$
-  SELECT info->>'hnr';
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_street_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'street')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_place_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'place_match')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_place_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'place_search')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-DROP TYPE IF EXISTS token_addresstoken CASCADE;
-CREATE TYPE token_addresstoken AS (
-  key TEXT,
-  match_tokens INT[],
-  search_tokens INT[]
-);
-
-CREATE OR REPLACE FUNCTION token_get_address_tokens(info JSONB)
-  RETURNS SETOF token_addresstoken
-AS $$
-  SELECT key, (value->>1)::int[] as match_tokens,
-         (value->>0)::int[] as search_tokens
-  FROM jsonb_each(info->'addr');
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_normalized_postcode(postcode TEXT)
-  RETURNS TEXT
-AS $$
-  SELECT CASE WHEN postcode SIMILAR TO '%(,|;)%' THEN NULL ELSE upper(trim(postcode))END;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return token info that should be saved permanently in the database.
-CREATE OR REPLACE FUNCTION token_strip_info(info JSONB)
-  RETURNS JSONB
-AS $$
-  SELECT NULL::JSONB;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
---------------- private functions ----------------------------------------------
-
-CREATE OR REPLACE FUNCTION getorcreate_full_word(norm_term TEXT, lookup_terms TEXT[],
-                                                 OUT full_token INT,
-                                                 OUT partial_tokens INT[])
-  AS $$
-DECLARE
-  partial_terms TEXT[] = '{}'::TEXT[];
-  term TEXT;
-  term_id INTEGER;
-  term_count INTEGER;
-BEGIN
-  SELECT min(word_id) INTO full_token
-    FROM word WHERE word = norm_term and type = 'W';
-
-  IF full_token IS NULL THEN
-    full_token := nextval('seq_word');
-    INSERT INTO word (word_id, word_token, type, word, info)
-      SELECT full_token, lookup_term, 'W', norm_term,
-             json_build_object('count', 0)
-        FROM unnest(lookup_terms) as lookup_term;
-  END IF;
-
-  FOR term IN SELECT unnest(string_to_array(unnest(lookup_terms), ' ')) LOOP
-    term := trim(term);
-    IF NOT (ARRAY[term] <@ partial_terms) THEN
-      partial_terms := partial_terms || term;
-    END IF;
-  END LOOP;
-
-  partial_tokens := '{}'::INT[];
-  FOR term IN SELECT unnest(partial_terms) LOOP
-    SELECT min(word_id), max(info->>'count') INTO term_id, term_count
-      FROM word WHERE word_token = term and type = 'w';
-
-    IF term_id IS NULL THEN
-      term_id := nextval('seq_word');
-      term_count := 0;
-      INSERT INTO word (word_id, word_token, type, info)
-        VALUES (term_id, term, 'w', json_build_object('count', term_count));
-    END IF;
-
-    IF term_count < {{ max_word_freq }} THEN
-      partial_tokens := array_merge(partial_tokens, ARRAY[term_id]);
-    END IF;
-  END LOOP;
-END;
-$$
-LANGUAGE plpgsql;
-
-
-CREATE OR REPLACE FUNCTION getorcreate_hnr_id(lookup_term TEXT)
-  RETURNS INTEGER
-  AS $$
-DECLARE
-  return_id INTEGER;
-BEGIN
-  SELECT min(word_id) INTO return_id FROM word
-    WHERE word_token = lookup_term and type = 'H';
-
-  IF return_id IS NULL THEN
-    return_id := nextval('seq_word');
-    INSERT INTO word (word_id, word_token, type)
-      VALUES (return_id, lookup_term, 'H');
-  END IF;
-
-  RETURN return_id;
-END;
-$$
-LANGUAGE plpgsql;

lib-sql/tokenizer/legacy_tokenizer.sql

@@ -1,393 +0,0 @@
--- Get tokens used for searching the given place.
---
--- These are the tokens that will be saved in the search_name table.
-CREATE OR REPLACE FUNCTION token_get_name_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'names')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Get tokens for matching the place name against others.
---
--- This should usually be restricted to full name tokens.
-CREATE OR REPLACE FUNCTION token_get_name_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'names')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return the housenumber tokens applicable for the place.
-CREATE OR REPLACE FUNCTION token_get_housenumber_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'hnr_tokens')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return the housenumber in the form that it can be matched during search.
-CREATE OR REPLACE FUNCTION token_normalized_housenumber(info JSONB)
-  RETURNS TEXT
-AS $$
-  SELECT info->>'hnr';
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_street_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'street')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_place_match_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'place_match')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_addr_place_search_tokens(info JSONB)
-  RETURNS INTEGER[]
-AS $$
-  SELECT (info->>'place_search')::INTEGER[]
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-DROP TYPE IF EXISTS token_addresstoken CASCADE;
-CREATE TYPE token_addresstoken AS (
-  key TEXT,
-  match_tokens INT[],
-  search_tokens INT[]
-);
-
-CREATE OR REPLACE FUNCTION token_get_address_tokens(info JSONB)
-  RETURNS SETOF token_addresstoken
-AS $$
-  SELECT key, (value->>1)::int[] as match_tokens,
-         (value->>0)::int[] as search_tokens
-  FROM jsonb_each(info->'addr');
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION token_normalized_postcode(postcode TEXT)
-  RETURNS TEXT
-AS $$
-  SELECT CASE WHEN postcode SIMILAR TO '%(,|;)%' THEN NULL ELSE upper(trim(postcode))END;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
-
--- Return token info that should be saved permanently in the database.
-CREATE OR REPLACE FUNCTION token_strip_info(info JSONB)
-  RETURNS JSONB
-AS $$
-  SELECT NULL::JSONB;
-$$ LANGUAGE SQL IMMUTABLE STRICT;
-
---------------- private functions ----------------------------------------------
-
--- Functions for term normalisation and access to the 'word' table.
-
-CREATE OR REPLACE FUNCTION transliteration(text) RETURNS text
-  AS '{{ modulepath }}/nominatim.so', 'transliteration'
-LANGUAGE c IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION gettokenstring(text) RETURNS text
-  AS '{{ modulepath }}/nominatim.so', 'gettokenstring'
-LANGUAGE c IMMUTABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION make_standard_name(name TEXT) RETURNS TEXT
-  AS $$
-DECLARE
-  o TEXT;
-BEGIN
-  o := public.gettokenstring(public.transliteration(name));
-  RETURN trim(substr(o,1,length(o)));
-END;
-$$
-LANGUAGE plpgsql IMMUTABLE;
-
--- returns NULL if the word is too common
-CREATE OR REPLACE FUNCTION getorcreate_word_id(lookup_word TEXT) 
-  RETURNS INTEGER
-  AS $$
-DECLARE
-  lookup_token TEXT;
-  return_word_id INTEGER;
-  count INTEGER;
-BEGIN
-  lookup_token := trim(lookup_word);
-  SELECT min(word_id), max(search_name_count) FROM word
-    WHERE word_token = lookup_token and class is null and type is null
-    INTO return_word_id, count;
-  IF return_word_id IS NULL THEN
-    return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, null, null, 0);
-  ELSE
-    IF count > {{ max_word_freq }} THEN
-      return_word_id := NULL;
-    END IF;
-  END IF;
-  RETURN return_word_id;
-END;
-$$
-LANGUAGE plpgsql;
-
-
--- Create housenumber tokens from an OSM addr:housenumber.
--- The housnumber is split at comma and semicolon as necessary.
--- The function returns the normalized form of the housenumber suitable
--- for comparison.
-CREATE OR REPLACE FUNCTION create_housenumbers(housenumbers TEXT[],
-                                               OUT tokens TEXT,
-                                               OUT normtext TEXT)
-  AS $$
-BEGIN
-  SELECT array_to_string(array_agg(trans), ';'), array_agg(tid)::TEXT
-    INTO normtext, tokens
-    FROM (SELECT lookup_word as trans, getorcreate_housenumber_id(lookup_word) as tid
-          FROM (SELECT make_standard_name(h) as lookup_word
-                FROM unnest(housenumbers) h) x) y;
-END;
-$$ LANGUAGE plpgsql STABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION getorcreate_housenumber_id(lookup_word TEXT)
-  RETURNS INTEGER
-  AS $$
-DECLARE
-  lookup_token TEXT;
-  return_word_id INTEGER;
-BEGIN
-  lookup_token := ' ' || trim(lookup_word);
-  SELECT min(word_id) FROM word
-    WHERE word_token = lookup_token and class='place' and type='house'
-    INTO return_word_id;
-  IF return_word_id IS NULL THEN
-    return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, null,
-                             'place', 'house', null, 0);
-  END IF;
-  RETURN return_word_id;
-END;
-$$
-LANGUAGE plpgsql;
-
-
-CREATE OR REPLACE FUNCTION create_postcode_id(postcode TEXT)
-  RETURNS BOOLEAN
-  AS $$
-DECLARE
-  r RECORD;
-  lookup_token TEXT;
-  return_word_id INTEGER;
-BEGIN
-  lookup_token := ' ' || make_standard_name(postcode);
-  FOR r IN
-    SELECT word_id FROM word
-    WHERE word_token = lookup_token and word = postcode
-          and class='place' and type='postcode'
-  LOOP
-    RETURN false;
-  END LOOP;
-
-  INSERT INTO word VALUES (nextval('seq_word'), lookup_token, postcode,
-                           'place', 'postcode', null, 0);
-  RETURN true;
-END;
-$$
-LANGUAGE plpgsql;
-
-
-CREATE OR REPLACE FUNCTION getorcreate_name_id(lookup_word TEXT, src_word TEXT)
-  RETURNS INTEGER
-  AS $$
-DECLARE
-  lookup_token TEXT;
-  nospace_lookup_token TEXT;
-  return_word_id INTEGER;
-BEGIN
-  lookup_token := ' '||trim(lookup_word);
-  SELECT min(word_id) FROM word
-  WHERE word_token = lookup_token and class is null and type is null
-  INTO return_word_id;
-  IF return_word_id IS NULL THEN
-    return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, src_word,
-                             null, null, null, 0);
-  END IF;
-  RETURN return_word_id;
-END;
-$$
-LANGUAGE plpgsql;
-
-
--- Normalize a string and lookup its word ids (partial words).
-CREATE OR REPLACE FUNCTION addr_ids_from_name(lookup_word TEXT)
-  RETURNS INTEGER[]
-  AS $$
-DECLARE
-  words TEXT[];
-  id INTEGER;
-  return_word_id INTEGER[];
-  word_ids INTEGER[];
-  j INTEGER;
-BEGIN
-  words := string_to_array(make_standard_name(lookup_word), ' ');
-  IF array_upper(words, 1) IS NOT NULL THEN
-    FOR j IN 1..array_upper(words, 1) LOOP
-      IF (words[j] != '') THEN
-        SELECT array_agg(word_id) INTO word_ids
-          FROM word
-         WHERE word_token = words[j] and class is null and type is null;
-
-        IF word_ids IS NULL THEN
-          id := nextval('seq_word');
-          INSERT INTO word VALUES (id, words[j], null, null, null, null, 0);
-          return_word_id := return_word_id || id;
-        ELSE
-          return_word_id := array_merge(return_word_id, word_ids);
-        END IF;
-      END IF;
-    END LOOP;
-  END IF;
-
-  RETURN return_word_id;
-END;
-$$
-LANGUAGE plpgsql;
-
-
--- Normalize a string and look up its name ids (full words).
-CREATE OR REPLACE FUNCTION word_ids_from_name(lookup_word TEXT)
-  RETURNS INTEGER[]
-  AS $$
-DECLARE
-  lookup_token TEXT;
-  return_word_ids INTEGER[];
-BEGIN
-  lookup_token := ' '|| make_standard_name(lookup_word);
-  SELECT array_agg(word_id) FROM word
-    WHERE word_token = lookup_token and class is null and type is null
-    INTO return_word_ids;
-  RETURN return_word_ids;
-END;
-$$
-LANGUAGE plpgsql STABLE STRICT;
-
-
-CREATE OR REPLACE FUNCTION make_keywords(src HSTORE)
-  RETURNS INTEGER[]
-  AS $$
-DECLARE
-  result INTEGER[];
-  s TEXT;
-  w INTEGER;
-  words TEXT[];
-  value TEXT;
-  j INTEGER;
-BEGIN
-  result := '{}'::INTEGER[];
-
-  FOR value IN SELECT unnest(regexp_split_to_array(svals(src), E'[,;]')) LOOP
-    -- full name
-    s := make_standard_name(value);
-    w := getorcreate_name_id(s, value);
-
-    IF not(ARRAY[w] <@ result) THEN
-      result := result || w;
-    END IF;
-
-    -- partial single-word terms
-    words := string_to_array(s, ' ');
-    IF array_upper(words, 1) IS NOT NULL THEN
-      FOR j IN 1..array_upper(words, 1) LOOP
-        IF (words[j] != '') THEN
-          w = getorcreate_word_id(words[j]);
-          IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
-            result := result || w;
-          END IF;
-        END IF;
-      END LOOP;
-    END IF;
-
-    -- consider parts before an opening braket a full word as well
-    words := regexp_split_to_array(value, E'[(]');
-    IF array_upper(words, 1) > 1 THEN
-      s := make_standard_name(words[1]);
-      IF s != '' THEN
-        w := getorcreate_name_id(s, words[1]);
-        IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
-          result := result || w;
-        END IF;
-      END IF;
-    END IF;
-
-    s := regexp_replace(value, '市$', '');
-    IF s != value THEN
-      s := make_standard_name(s);
-      IF s != '' THEN
-        w := getorcreate_name_id(s, value);
-        IF NOT (ARRAY[w] <@ result) THEN
-          result := result || w;
-        END IF;
-      END IF;
-    END IF;
-
-  END LOOP;
-
-  RETURN result;
-END;
-$$
-LANGUAGE plpgsql;
-
-
-CREATE OR REPLACE FUNCTION precompute_words(src TEXT)
-  RETURNS INTEGER
-  AS $$
-DECLARE
-  s TEXT;
-  w INTEGER;
-  words TEXT[];
-  i INTEGER;
-  j INTEGER;
-BEGIN
-  s := make_standard_name(src);
-  w := getorcreate_name_id(s, src);
-
-  w := getorcreate_word_id(s);
-
-  words := string_to_array(s, ' ');
-  IF array_upper(words, 1) IS NOT NULL THEN
-    FOR j IN 1..array_upper(words, 1) LOOP
-      IF (words[j] != '') THEN
-        w := getorcreate_word_id(words[j]);
-      END IF;
-    END LOOP;
-  END IF;
-
-  words := regexp_split_to_array(src, E'[,;()]');
-  IF array_upper(words, 1) != 1 THEN
-    FOR j IN 1..array_upper(words, 1) LOOP
-      s := make_standard_name(words[j]);
-      IF s != '' THEN
-        w := getorcreate_word_id(s);
-      END IF;
-    END LOOP;
-  END IF;
-
-  s := regexp_replace(src, '市$', '');
-  IF s != src THEN
-    s := make_standard_name(s);
-    IF s != '' THEN
-      w := getorcreate_name_id(s, src);
-    END IF;
-  END IF;
-
-  RETURN 1;
-END;
-$$
-LANGUAGE plpgsql;

lib-sql/tokenizer/legacy_tokenizer_indices.sql

@@ -1,2 +0,0 @@
-CREATE INDEX IF NOT EXISTS idx_word_word_id
-  ON word USING BTREE (word_id) {{db.tablespace.search_index}};

lib-sql/tokenizer/legacy_tokenizer_tables.sql

@@ -1,21 +0,0 @@
-DROP TABLE IF EXISTS word;
-CREATE TABLE word (
-  word_id INTEGER,
-  word_token text NOT NULL,
-  word text,
-  class text,
-  type text,
-  country_code varchar(2),
-  search_name_count INTEGER,
-  operator TEXT
-) {{db.tablespace.search_data}};
-
-CREATE INDEX idx_word_word_token ON word
-    USING BTREE (word_token) {{db.tablespace.search_index}};
-CREATE INDEX idx_word_word ON word
-    USING BTREE (word) {{db.tablespace.search_index}} WHERE word is not null;
-GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}";
-
-DROP SEQUENCE IF EXISTS seq_word;
-CREATE SEQUENCE seq_word start 1;
-GRANT SELECT ON seq_word to "{{config.DATABASE_WEBUSER}}";

lib-sql/update-postcodes.sql

@@ -0,0 +1,58 @@
+-- Create a temporary table with postcodes from placex.
+
+CREATE TEMP TABLE tmp_new_postcode_locations AS
+SELECT country_code,
+       upper(trim (both ' ' from address->'postcode')) as pc,
+       ST_Centroid(ST_Collect(ST_Centroid(geometry))) as centroid
+  FROM placex
+ WHERE address ? 'postcode'
+       AND address->'postcode' NOT SIMILAR TO '%(,|;|:)%'
+       AND geometry IS NOT null
+GROUP BY country_code, pc;
+
+CREATE INDEX idx_tmp_new_postcode_locations
+          ON tmp_new_postcode_locations (pc, country_code);
+
+-- add extra US postcodes
+INSERT INTO tmp_new_postcode_locations (country_code, pc, centroid)
+    SELECT 'us', postcode, ST_SetSRID(ST_Point(x,y),4326)
+      FROM us_postcode u
+      WHERE NOT EXISTS (SELECT 0 FROM tmp_new_postcode_locations new
+                         WHERE new.country_code = 'us' AND new.pc = u.postcode);
+-- add extra UK postcodes
+INSERT INTO tmp_new_postcode_locations (country_code, pc, centroid)
+    SELECT 'gb', postcode, geometry FROM gb_postcode g
+     WHERE NOT EXISTS (SELECT 0 FROM tmp_new_postcode_locations new
+                             WHERE new.country_code = 'gb' and new.pc = g.postcode);
+
+-- Remove all postcodes that are no longer valid
+DELETE FROM location_postcode old
+  WHERE NOT EXISTS(SELECT 0 FROM tmp_new_postcode_locations new
+                   WHERE old.postcode = new.pc
+                         AND old.country_code = new.country_code);
+
+-- Update geometries where necessary
+UPDATE location_postcode old SET geometry = new.centroid, indexed_status = 1
+  FROM tmp_new_postcode_locations new
+ WHERE old.postcode = new.pc AND old.country_code = new.country_code
+       AND ST_AsText(old.geometry) != ST_AsText(new.centroid);
+
+-- Remove all postcodes that already exist from the temporary table
+DELETE FROM tmp_new_postcode_locations new
+    WHERE EXISTS(SELECT 0 FROM location_postcode old
+                 WHERE old.postcode = new.pc AND old.country_code = new.country_code);
+
+-- Add newly added postcode
+INSERT INTO location_postcode
+  (place_id, indexed_status, country_code, postcode, geometry)
+  SELECT nextval('seq_place'), 1, country_code, pc, centroid
+    FROM tmp_new_postcode_locations new;
+
+-- Remove unused word entries
+DELETE FROM word
+    WHERE class = 'place' AND type = 'postcode'
+          AND NOT EXISTS (SELECT 0 FROM location_postcode p
+                          WHERE p.postcode = word.word);
+
+-- Finally index the newly inserted postcodes
+UPDATE location_postcode SET indexed_status = 0 WHERE indexed_status > 0;

lib-sql/words.sql

@@ -0,0 +1,14 @@
+CREATE TABLE word_frequencies AS
+  (SELECT unnest(make_keywords(v)) as id, sum(count) as count
+     FROM (select svals(name) as v, count(*)from place group by v) cnt
+    WHERE v is not null
+ GROUP BY id);
+
+select count(getorcreate_postcode_id(v)) from (select distinct address->'postcode' as v from place where address ? 'postcode') as w where v is not null;
+select count(create_housenumber_id(v)) from (select distinct address->'housenumber' as v from place where address ? 'housenumber') as w;
+
+-- copy the word frequencies
+update word set search_name_count = count from word_frequencies wf where wf.id = word.word_id;
+
+-- and drop the temporary frequency table again
+drop table word_frequencies;

manual/nominatim.1

@@ -3,10 +3,12 @@
 nominatim
 .SH SYNOPSIS
 .B nominatim
-[-h] {import,freeze,replication,special-phrases,add-data,index,refresh,admin,export,serve,search,reverse,lookup,details,status} ...
+[-h] {import,freeze,replication,special-phrases,add-data,index,refresh,admin,export,serve,search,reverse,lookup,details,status,transition} ...
 .SH DESCRIPTION
     Command\-line tools for importing, updating, administrating and
+.br
     querying the Nominatim database.
+.br
     
 .SH OPTIONS
 
@@ -58,6 +60,9 @@ nominatim
 .TP
 \fBnominatim\fR \fI\,status\/\fR
     Execute API status query.
+.TP
+\fBnominatim\fR \fI\,transition\/\fR
+    Internal functions for code transition. Do not use.
 .SH OPTIONS 'nominatim import'
 usage: nominatim import [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                         (--osm-file FILE | --continue {load-data,indexing,db-postprocess})
@@ -66,6 +71,7 @@ usage: nominatim import [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                         [--index-noanalyse]
 
     Create a new Nominatim database from an OSM file.
+.br
     
 
 
@@ -122,13 +128,21 @@ Do not perform analyse operations during index
 usage: nominatim freeze [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
 
     Make database read\-only.
+.br
 
+.br
     About half of data in the Nominatim database is kept only to be able to
+.br
     keep the data up\-to\-date with new changes made in OpenStreetMap. This
+.br
     command drops all this data and only keeps the part needed for geocoding
+.br
     itself.
+.br
 
+.br
     This command has the same effect as the `\-\-no\-updates` option for imports.
+.br
     
 
 
@@ -157,6 +171,7 @@ usage: nominatim replication [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                              [--socket-timeout SOCKET_TIMEOUT]
 
     Update the database using an online replication service.
+.br
     
 
 
@@ -210,6 +225,7 @@ usage: nominatim special-phrases [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                                  [--import-from-wiki]
 
     Import special phrases.
+.br
     
 
 
@@ -240,9 +256,13 @@ usage: nominatim add-data [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                           [--use-main-api]
 
     Add additional data from a file or an online source.
+.br
 
+.br
     Data is only imported, not indexed. You need to call `nominatim index`
+.br
     to complete the process.
+.br
     
 
 
@@ -297,6 +317,7 @@ usage: nominatim index [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                        [--maxrank RANK]
 
     Reindex all new and modified data.
+.br
     
 
 
@@ -341,8 +362,11 @@ usage: nominatim refresh [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                          [--enable-debug-statements]
 
     Recompute auxiliary data used by the indexing process.
+.br
 
+.br
     These functions must not be run in parallel with other update commands.
+.br
     
 
 
@@ -406,6 +430,7 @@ usage: nominatim admin [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                        [--osm-id OSM_ID | --place-id PLACE_ID]
 
     Analyse and maintain the database.
+.br
     
 
 
@@ -468,6 +493,7 @@ usage: nominatim export [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                         [--restrict-to-osm-relation ID]
 
     Export addresses as CSV file from the database.
+.br
     
 
 
@@ -525,12 +551,19 @@ usage: nominatim serve [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                        [--server SERVER]
 
     Start a simple web server for serving the API.
+.br
 
+.br
     This command starts the built\-in PHP webserver to serve the website
+.br
     from the current project directory. This webserver is only suitable
+.br
     for testing and develop. Do not use it in production setups!
+.br
 
+.br
     By the default, the webserver can be accessed at: http://127.0.0.1:8088
+.br
     
 
 
@@ -569,6 +602,7 @@ usage: nominatim search [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                         [--viewbox X1,Y1,X2,Y2] [--bounded] [--no-dedupe]
 
     Execute API search query.
+.br
     
 
 
@@ -679,6 +713,7 @@ usage: nominatim reverse [-h] [-q] [-v] [--project-dir DIR] [-j NUM] --lat LAT
                          [--polygon-threshold TOLERANCE]
 
     Execute API reverse query.
+.br
     
 
 
@@ -748,6 +783,7 @@ usage: nominatim lookup [-h] [-q] [-v] [--project-dir DIR] [-j NUM] --id OSMID
                         [--polygon-threshold TOLERANCE]
 
     Execute API lookup query.
+.br
     
 
 
@@ -809,6 +845,7 @@ usage: nominatim details [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                          [--lang LANGS]
 
     Execute API details query.
+.br
     
 
 
@@ -882,6 +919,7 @@ usage: nominatim status [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
                         [--format {text,json}]
 
     Execute API status query.
+.br
     
 
 
@@ -906,6 +944,107 @@ Number of parallel threads to use
 \fB\-\-format\fR {text,json}
 Format of result
 
+.SH OPTIONS 'nominatim transition'
+usage: nominatim transition [-h] [-q] [-v] [--project-dir DIR] [-j NUM]
+                            [--create-db] [--setup-db] [--import-data]
+                            [--load-data] [--create-tables]
+                            [--create-partition-tables] [--index]
+                            [--create-search-indices] [--create-country-names]
+                            [--no-partitions] [--osm-file FILE] [--drop]
+                            [--osm2pgsql-cache SIZE] [--no-analyse]
+                            [--ignore-errors] [--reverse-only]
+                            [--tiger-data FILE]
+
+    Internal functions for code transition. Do not use.
+.br
+    
+
+
+
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+Print only error messages
+
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Increase verboseness of output
+
+.TP
+\fB\-\-project\-dir\fR DIR
+Base directory of the Nominatim installation (default:.)
+
+.TP
+\fB\-j\fR NUM, \fB\-\-threads\fR NUM
+Number of parallel threads to use
+
+.TP
+\fB\-\-create\-db\fR
+Create nominatim db
+
+.TP
+\fB\-\-setup\-db\fR
+Build a blank nominatim db
+
+.TP
+\fB\-\-import\-data\fR
+Import a osm file
+
+.TP
+\fB\-\-load\-data\fR
+Copy data to live tables from import table
+
+.TP
+\fB\-\-create\-tables\fR
+Create main tables
+
+.TP
+\fB\-\-create\-partition\-tables\fR
+Create required partition tables
+
+.TP
+\fB\-\-index\fR
+Index the data
+
+.TP
+\fB\-\-create\-search\-indices\fR
+Create additional indices required for search and update
+
+.TP
+\fB\-\-create\-country\-names\fR
+Create search index for default country names.
+
+.TP
+\fB\-\-no\-partitions\fR
+Do not partition search indices
+
+.TP
+\fB\-\-osm\-file\fR FILE
+File to import
+
+.TP
+\fB\-\-drop\fR
+Drop tables needed for updates, making the database readonly
+
+.TP
+\fB\-\-osm2pgsql\-cache\fR SIZE
+Size of cache to be used by osm2pgsql (in MB)
+
+.TP
+\fB\-\-no\-analyse\fR
+Do not perform analyse operations during index
+
+.TP
+\fB\-\-ignore\-errors\fR
+Ignore certain erros on import.
+
+.TP
+\fB\-\-reverse\-only\fR
+Do not create search tables and indexes
+
+.TP
+\fB\-\-tiger\-data\fR FILE
+File to import
+
 .SH DISTRIBUTION
 The latest version of Nominatim may be downloaded from
 .UR https://nominatim.org

nominatim/cli.py

@@ -8,11 +8,12 @@ import sys
 import argparse
 from pathlib import Path
 
-from nominatim.config import Configuration
-from nominatim.tools.exec_utils import run_legacy_script, run_php_server
-from nominatim.errors import UsageError
-from nominatim import clicmd
-from nominatim.clicmd.args import NominatimArgs
+from .config import Configuration
+from .tools.exec_utils import run_legacy_script, run_php_server
+from .errors import UsageError
+from . import clicmd
+from .clicmd.args import NominatimArgs
+from .tools import tiger_data
 
 LOG = logging.getLogger()
 
@@ -83,11 +84,6 @@ class CommandlineParser:
 
         args.config = Configuration(args.project_dir, args.config_dir,
                                     environ=kwargs.get('environ', os.environ))
-        args.config.set_libdirs(module=args.module_dir,
-                                osm2pgsql=args.osm2pgsql_path,
-                                php=args.phplib_dir,
-                                sql=args.sqllib_dir,
-                                data=args.data_dir)
 
         log = logging.getLogger()
         log.warning('Using project directory: %s', str(args.project_dir))
@@ -103,7 +99,7 @@ class CommandlineParser:
         return 1
 
 
-# Subcommand classes
+##### Subcommand classes
 #
 # Each class needs to implement two functions: add_args() adds the CLI parameters
 # for the subfunction, run() executes the subcommand.
@@ -114,6 +110,61 @@ class CommandlineParser:
 #
 # No need to document the functions each time.
 # pylint: disable=C0111
+# Using non-top-level imports to make pyosmium optional for replication only.
+# pylint: disable=E0012,C0415
+class UpdateAddData:
+    """\
+    Add additional data from a file or an online source.
+
+    Data is only imported, not indexed. You need to call `nominatim index`
+    to complete the process.
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group_name = parser.add_argument_group('Source')
+        group = group_name.add_mutually_exclusive_group(required=True)
+        group.add_argument('--file', metavar='FILE',
+                           help='Import data from an OSM file')
+        group.add_argument('--diff', metavar='FILE',
+                           help='Import data from an OSM diff file')
+        group.add_argument('--node', metavar='ID', type=int,
+                           help='Import a single node from the API')
+        group.add_argument('--way', metavar='ID', type=int,
+                           help='Import a single way from the API')
+        group.add_argument('--relation', metavar='ID', type=int,
+                           help='Import a single relation from the API')
+        group.add_argument('--tiger-data', metavar='DIR',
+                           help='Add housenumbers from the US TIGER census database.')
+        group = parser.add_argument_group('Extra arguments')
+        group.add_argument('--use-main-api', action='store_true',
+                           help='Use OSM API instead of Overpass to download objects')
+
+    @staticmethod
+    def run(args):
+        if args.tiger_data:
+            return tiger_data.add_tiger_data(args.config.get_libpq_dsn(),
+                                             args.tiger_data,
+                                             args.threads or 1,
+                                             args.config,
+                                             args.sqllib_dir)
+
+        params = ['update.php']
+        if args.file:
+            params.extend(('--import-file', args.file))
+        elif args.diff:
+            params.extend(('--import-diff', args.diff))
+        elif args.node:
+            params.extend(('--import-node', args.node))
+        elif args.way:
+            params.extend(('--import-way', args.way))
+        elif args.relation:
+            params.extend(('--import-relation', args.relation))
+        if args.use_main_api:
+            params.append('--use-main-api')
+        return run_legacy_script(*params, nominatim_env=args)
+
+
 class QueryExport:
     """\
     Export addresses as CSV file from the database.
@@ -204,9 +255,9 @@ def get_set_parser(**kwargs):
 
     parser.add_subcommand('special-phrases', clicmd.ImportSpecialPhrases)
 
-    parser.add_subcommand('add-data', clicmd.UpdateAddData)
+    parser.add_subcommand('add-data', UpdateAddData)
     parser.add_subcommand('index', clicmd.UpdateIndex)
-    parser.add_subcommand('refresh', clicmd.UpdateRefresh())
+    parser.add_subcommand('refresh', clicmd.UpdateRefresh)
 
     parser.add_subcommand('admin', clicmd.AdminFuncs)
 
@@ -222,6 +273,8 @@ def get_set_parser(**kwargs):
     else:
         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 
+    parser.add_subcommand('transition', clicmd.AdminTransition)
+
     return parser
 
 

nominatim/clicmd/__init__.py

@@ -2,12 +2,12 @@
 Subcommand definitions for the command-line tool.
 """
 
-from nominatim.clicmd.setup import SetupAll
-from nominatim.clicmd.replication import UpdateReplication
-from nominatim.clicmd.api import APISearch, APIReverse, APILookup, APIDetails, APIStatus
-from nominatim.clicmd.index import UpdateIndex
-from nominatim.clicmd.refresh import UpdateRefresh
-from nominatim.clicmd.add_data import UpdateAddData
-from nominatim.clicmd.admin import AdminFuncs
-from nominatim.clicmd.freeze import SetupFreeze
-from nominatim.clicmd.special_phrases import ImportSpecialPhrases
+from .setup import SetupAll
+from .replication import UpdateReplication
+from .api import APISearch, APIReverse, APILookup, APIDetails, APIStatus
+from .index import UpdateIndex
+from .refresh import UpdateRefresh
+from .admin import AdminFuncs
+from .freeze import SetupFreeze
+from .transition import AdminTransition
+from .special_phrases import ImportSpecialPhrases

nominatim/clicmd/add_data.py

@@ -1,76 +0,0 @@
-"""
-Implementation of the 'add-data' subcommand.
-"""
-import logging
-
-# Do not repeat documentation of subcommand classes.
-# pylint: disable=C0111
-# Using non-top-level imports to avoid eventually unused imports.
-# pylint: disable=E0012,C0415
-
-LOG = logging.getLogger()
-
-class UpdateAddData:
-    """\
-    Add additional data from a file or an online source.
-
-    Data is only imported, not indexed. You need to call `nominatim index`
-    to complete the process.
-    """
-
-    @staticmethod
-    def add_args(parser):
-        group_name = parser.add_argument_group('Source')
-        group = group_name.add_mutually_exclusive_group(required=True)
-        group.add_argument('--file', metavar='FILE',
-                           help='Import data from an OSM file or diff file')
-        group.add_argument('--diff', metavar='FILE',
-                           help='Import data from an OSM diff file (deprecated: use --file)')
-        group.add_argument('--node', metavar='ID', type=int,
-                           help='Import a single node from the API')
-        group.add_argument('--way', metavar='ID', type=int,
-                           help='Import a single way from the API')
-        group.add_argument('--relation', metavar='ID', type=int,
-                           help='Import a single relation from the API')
-        group.add_argument('--tiger-data', metavar='DIR',
-                           help='Add housenumbers from the US TIGER census database.')
-        group = parser.add_argument_group('Extra arguments')
-        group.add_argument('--use-main-api', action='store_true',
-                           help='Use OSM API instead of Overpass to download objects')
-        group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
-                           help='Size of cache to be used by osm2pgsql (in MB)')
-        group.add_argument('--socket-timeout', dest='socket_timeout', type=int, default=60,
-                           help='Set timeout for file downloads.')
-
-    @staticmethod
-    def run(args):
-        from nominatim.tokenizer import factory as tokenizer_factory
-        from nominatim.tools import tiger_data, add_osm_data
-
-        if args.tiger_data:
-            tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
-            return tiger_data.add_tiger_data(args.tiger_data,
-                                             args.config, args.threads or 1,
-                                             tokenizer)
-
-        osm2pgsql_params = args.osm2pgsql_options(default_cache=1000, default_threads=1)
-        if args.file or args.diff:
-            return add_osm_data.add_data_from_file(args.file or args.diff,
-                                                   osm2pgsql_params)
-
-        if args.node:
-            return add_osm_data.add_osm_object('node', args.node,
-                                               args.use_main_api,
-                                               osm2pgsql_params)
-
-        if args.way:
-            return add_osm_data.add_osm_object('way', args.way,
-                                               args.use_main_api,
-                                               osm2pgsql_params)
-
-        if args.relation:
-            return add_osm_data.add_osm_object('relation', args.relation,
-                                               args.use_main_api,
-                                               osm2pgsql_params)
-
-        return 0

nominatim/clicmd/admin.py

@@ -3,8 +3,8 @@ Implementation of the 'admin' subcommand.
 """
 import logging
 
-from nominatim.tools.exec_utils import run_legacy_script
-from nominatim.db.connection import connect
+from ..tools.exec_utils import run_legacy_script
+from ..db.connection import connect
 
 # Do not repeat documentation of subcommand classes.
 # pylint: disable=C0111

nominatim/clicmd/api.py

@@ -3,7 +3,7 @@ Subcommand definitions for API calls from the command line.
 """
 import logging
 
-from nominatim.tools.exec_utils import run_api_script
+from ..tools.exec_utils import run_api_script
 
 # Do not repeat documentation of subcommand classes.
 # pylint: disable=C0111
@@ -90,7 +90,7 @@ class APISearch:
         if args.query:
             params = dict(q=args.query)
         else:
-            params = {k: getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
+            params = {k : getattr(args, k) for k, _ in STRUCTURED_QUERY if getattr(args, k)}
 
         for param, _ in EXTRADATA_PARAMS:
             if getattr(args, param):