prepare for release 3.1.0

Switch to mkdocs for building documentation

CMakeLists.txt

@@ -19,8 +19,8 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 3)
-set(NOMINATIM_VERSION_MINOR 0)
+set(NOMINATIM_VERSION_MINOR 1)
-set(NOMINATIM_VERSION_PATCH 1)
+set(NOMINATIM_VERSION_PATCH 0)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
 

ChangeLog

@@ -1,3 +1,23 @@
+3.1.0
+
+ * rework postcode handling and introduce location_postcode table
+ * make setup less verbose and print a summary in the end
+ * setup: error out when web site user does not exist
+ * add more API tests to complete code coverage
+ * reinstate key-value amenity search (special term [key=value])
+ * fix detection of coordinates in query
+ * various smaller tweaks to ranking of search interpretations
+ * complete overhaul of PHP frontend code using OOP
+ * add address rank to details page
+ * update Tiger scripts for 2017 data and clean up unused code
+ * various bug fixes and improvements to UI
+ * improve reverse geocoding performance close to coasts
+ * more PHP style cleanup (quoting)
+ * allow unnamed road in reverse geocoding to avoid too far off results
+ * add function to recalculate counts for full-word search term
+ * add function to check if new updates are available
+ * update documentation and switch to mkdocs for generating HTML
+
 3.0.1
 
  * fix bug in geometry building algorithm in osm2pgsql

README.md

@@ -6,7 +6,7 @@ Nominatim
 Nominatim (from the Latin, 'by name') is a tool to search OpenStreetMap data
 by name and address (geocoding) and to generate synthetic addresses of
 OSM points (reverse geocoding). An instance with up-to-date data can be found
-at http://nominatim.openstreetmap.org. Nominatim is also used as one of the
+at https://nominatim.openstreetmap.org. Nominatim is also used as one of the
 sources for the Search box on the OpenStreetMap home page and powers the search
 on the MapQuest Open Initiative websites.
 
@@ -16,13 +16,18 @@ Documentation
 More information about Nominatim, including usage and installation instructions,
 can be found in the docs/ subdirectory and in the OSM wiki at:
 
-http://wiki.openstreetmap.org/wiki/Nominatim
+https://nominatim.org
 
 Installation
 ============
 
-There are detailed installation instructions in the /docs directory.
+The latest stable release can be downloaded from https://nominatim.org.
-Here is a quick summary of the necessary steps.
+There you can also find [installation instructions for the release](https://nominatim.org/release-docs/latest/Installation).
+
+Detailed installation instructions for the development version can be
+found in the `/docs` directory, see [docs/Installation.md](docs/Installation.md).
+
+A quick summary of the necessary steps:
 
 1. Compile Nominatim:
 
@@ -31,21 +36,13 @@ Here is a quick summary of the necessary steps.
         cmake ..
         make
 
-   For more detailed installation instructions see [docs/Installation.md](docs/Installation.md).
-   There are also step-by-step instructions for
-     [Ubuntu 16.04](docs/Install-on-Ubuntu-16.md) and
-     [CentOS 7.2](docs/Install-on-Centos-7.md).
-
 2. Get OSM data and import:
 
         ./build/utils/setup.php --osm-file <your planet file> --all
 
-   Details can be found in [docs/Import_and_update.md](docs/Import-and-Update.md)
-
 3. Point your webserver to the ./build/website directory.
 
 
-
 License
 =======
 
@@ -55,7 +52,7 @@ Contact and Bug reports
 ======================
 
 For questions you can join the geocoding mailinglist, see
-http://lists.openstreetmap.org/listinfo/geocoding
+https://lists.openstreetmap.org/listinfo/geocoding
 
 Bugs may be reported on the github project site:
 https://github.com/openstreetmap/Nominatim

VAGRANT.md

@@ -102,7 +102,7 @@ and then
 
 ## Running unit tests
 
-    cd ~/Nominatim/tests-php
+    cd ~/Nominatim/tests/php
     phpunit ./
 
 
@@ -128,7 +128,7 @@ No. Long running Nominatim installations will differ once new import features (o
 bug fixes) get added since those usually only get applied to new/changed data.
 
 Also this document skips the optional Wikipedia data import which affects ranking
-of search results. See [Nominatim installation](http://wiki.openstreetmap.org/wiki/Nominatim/Installation) for details.
+of search results. See [Nominatim installation](http://nominatim.org/release-docs/latest/Installation) for details.
 
 ##### Why Ubuntu and CentOS, can I test CentOS/CoreOS/FreeBSD?
 

data/words.sql

@@ -29787,7 +29787,6 @@ st	5557484
 -- prefill word table
 
 select count(make_keywords(v)) from (select distinct svals(name) as v from place) as w where v is not null;
-select count(make_keywords(v)) from (select distinct address->'postcode' as v from place where address ? 'postcode') as w where v is not null;
 select count(getorcreate_housenumber_id(make_standard_name(v))) from (select distinct address->'housenumber' as v from place where address ? 'housenumber') as w;
 
 -- copy the word frequencies

docs/CMakeLists.txt

@@ -1,32 +1,17 @@
 # Auto-generated vagrant install documentation
 
-set (INSTALLDOCFILES
-     Install-on-Centos-7
-     Install-on-Ubuntu-16
-    )
 
-foreach (df ${INSTALLDOCFILES})
+# build the actual documentation
-    ADD_CUSTOM_COMMAND( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${df}.md
-        COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/${df}.sh ${CMAKE_CURRENT_BINARY_DIR}/${df}.md
-        MAIN_DEPENDENCY ${PROJECT_SOURCE_DIR}/vagrant/${df}.sh
-        DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh
-        COMMENT "Creating markdown docs from vagrant/${df}.sh"
-        )
 
-ADD_CUSTOM_TARGET( md_install_${df} ALL
+configure_file(mkdocs.yml ../mkdocs.yml)
-        DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${df}.md
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/appendix)
-        )
-endforeach()
 
-# Copied static documentation
+ADD_CUSTOM_TARGET(doc
+   COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_SOURCE_DIR}/admin ${CMAKE_CURRENT_BINARY_DIR}/admin
+   COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_SOURCE_DIR}/index.md ${CMAKE_CURRENT_BINARY_DIR}/index.md
+   COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Centos-7.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Centos-7.md
+   COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-16.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-16.md
+   COMMAND mkdocs build -d ${CMAKE_CURRENT_BINARY_DIR}/../site-html -f ${CMAKE_CURRENT_BINARY_DIR}/../mkdocs.yml
+)
 
-set (GENERALDOCFILES
-     Installation.md
-     Import-and-Update.md
-     Faq.md
-    )
-
-foreach (df ${GENERALDOCFILES})
-   CONFIGURE_FILE(${df} ${df})
-endforeach()
 

docs/Install-on-Centos-7.md

@@ -1,181 +0,0 @@
-
-*Note:* these installation instructions are also available in executable
-        form for use with vagrant under `vagrant/Install-on-Centos-7.sh`.
-
-Installing the Required Software
-================================
-
-These instructions expect that you have a freshly installed CentOS version 7.
-Make sure all packages are up-to-date by running:
-
-    sudo yum update -y
-
-The standard CentOS repositories don't contain all the required packages,
-you need to enable the EPEL repository as well. To enable it on CentOS,
-install the epel-release RPM by running:
-
-    sudo yum install -y epel-release
-
-Now you can install all packages needed for Nominatim:
-
-    sudo yum install -y postgresql-server postgresql-contrib postgresql-devel postgis postgis-utils \
-                        git cmake make gcc gcc-c++ libtool policycoreutils-python \
-                        php-pgsql php php-pear php-pear-DB php-intl libpqxx-devel proj-epsg \
-                        bzip2-devel proj-devel geos-devel libxml2-devel boost-devel expat-devel zlib-devel
-
-If you want to run the test suite, you need to install the following
-additional packages:
-
-    sudo yum install -y python-pip python-Levenshtein python-psycopg2 \
-                        python-numpy php-phpunit-PHPUnit
-    pip install --user --upgrade pip setuptools lettuce==0.2.18 six==1.9 \
-                                 haversine Shapely pytidylib
-    sudo pear install PHP_CodeSniffer
-
-
-System Configuration
-====================
-
-The following steps are meant to configure a fresh CentOS installation
-for use with Nominatim. You may skip some of the steps if you have your
-OS already configured.
-
-Creating Dedicated User Accounts
---------------------------------
-
-Nominatim will run as a global service on your machine. It is therefore
-best to install it under its own separate user account. In the following
-we assume this user is called nominatim and the installation will be in
-/srv/nominatim. To create the user and directory run:
-
-    sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim
-
-You may find a more suitable location if you wish.
-
-To be able to copy and paste instructions from this manual, export
-user name and home directory now like this:
-
-    export USERNAME=nominatim
-    export USERHOME=/srv/nominatim
-
-**Never, ever run the installation as a root user.** You have been warned.
-
-Make sure that system servers can read from the home directory:
-
-    chmod a+x $USERHOME
-
-Setting up PostgreSQL
----------------------
-
-CentOS does not automatically create a database cluster. Therefore, start
-with initializing the database, then enable the server to start at boot:
-
-    sudo postgresql-setup initdb
-    sudo systemctl enable postgresql
-
-
-Next tune the postgresql configuration, which is located in 
-`/var/lib/pgsql/data/postgresql.conf`. See section *Postgres Tuning* in
-[the installation page](Installation.md) for the parameters to change.
-
-Now start the postgresql service after updating this config file.
-
-    sudo systemctl restart postgresql
-
-
-Finally, we need to add two postgres users: one for the user that does
-the import and another for the webserver which should access the database
-only for reading:
-
-
-    sudo -u postgres createuser -s $USERNAME
-    sudo -u postgres createuser apache
-
-
-Setting up the Apache Webserver
--------------------------------
-
-You need to create an alias to the website directory in your apache
-configuration. Add a separate nominatim configuration to your webserver:
-
-```
-sudo tee /etc/httpd/conf.d/nominatim.conf << EOFAPACHECONF
-<Directory "$USERHOME/Nominatim/build/website">
-  Options FollowSymLinks MultiViews
-  AddType text/html   .php
-  DirectoryIndex search.php
-  Require all granted
-</Directory>
-
-Alias /nominatim $USERHOME/Nominatim/build/website
-EOFAPACHECONF
-```
-
-
-
-
-Then reload apache
-
-
-    sudo systemctl restart httpd
-
-
-Adding SELinux Security Settings
---------------------------------
-
-It is a good idea to leave SELinux enabled and enforcing, particularly
-with a web server accessible from the Internet. At a minimum the
-following SELinux labeling should be done for Nominatim:
-
-    sudo semanage fcontext -a -t httpd_sys_content_t "$USERHOME/Nominatim/(website|lib|settings)(/.*)?"
-    sudo semanage fcontext -a -t lib_t "$USERHOME/Nominatim/module/nominatim.so"
-    sudo restorecon -R -v $USERHOME/Nominatim
-
-
-Installing Nominatim
-====================
-
-Building and Configuration
---------------------------
-
-Get the source code from Github and change into the source directory
-
-
-
-    cd $USERHOME
-    git clone --recursive git://github.com/openstreetmap/Nominatim.git
-    cd Nominatim
-
-
-
-
-
-When installing the latest source from github, you also need to
-download the country grid:
-
-
-    wget -O data/country_osm_grid.sql.gz http://www.nominatim.org/data/country_grid.sql.gz
-
-
-The code must be built in a separate directory. Create this directory,
-then configure and build Nominatim in there:
-
-    mkdir build
-    cd build
-    cmake $USERHOME/Nominatim
-    make
-
-You need to create a minimal configuration file that tells nominatim
-the name of your webserver user and the URL of the website:
-
-```
-tee settings/local.php << EOF
-<?php
- @define('CONST_Database_Web_User', 'apache');
- @define('CONST_Website_BaseURL', '/nominatim/');
-EOF
-```
-
-
-Nominatim is now ready to use. Continue with
-[importing a database from OSM data](Import-and-Update.md).

docs/Install-on-Ubuntu-16.md

@@ -1,167 +0,0 @@
-
-
-
-
-
-
-
-
-*Note:* these installation instructions are also available in executable
-        form for use with vagrant under vagrant/Install-on-Ubuntu-16.sh.
-
-Installing the Required Software
-================================
-
-These instructions expect that you have a freshly installed Ubuntu 16.04.
-
-Make sure all packages are are up-to-date by running:
-
-
-
-    sudo apt-get update -qq
-
-Now you can install all packages needed for Nominatim:
-
-    sudo apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \
-                            libboost-filesystem-dev libexpat1-dev zlib1g-dev libxml2-dev\
-                            libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev \
-                            postgresql-server-dev-9.5 postgresql-9.5-postgis-2.2 postgresql-contrib-9.5 \
-                            apache2 php php-pgsql libapache2-mod-php php-pear php-db \
-                            php-intl git
-
-If you want to run the test suite, you need to install the following
-additional packages:
-
-    sudo apt-get install -y python3-dev python3-pip python3-psycopg2 python3-tidylib phpunit
-
-    pip3 install --user behave nose # urllib3
-    sudo pear install PHP_CodeSniffer
-
-
-System Configuration
-====================
-
-The following steps are meant to configure a fresh Ubuntu installation
-for use with Nominatim. You may skip some of the steps if you have your
-OS already configured.
-
-Creating Dedicated User Accounts
---------------------------------
-
-Nominatim will run as a global service on your machine. It is therefore
-best to install it under its own separate user account. In the following
-we assume this user is called nominatim and the installation will be in
-/srv/nominatim. To create the user and directory run:
-
-    sudo useradd -d /srv/nominatim -s /bin/bash -m nominatim
-
-You may find a more suitable location if you wish.
-
-To be able to copy and paste instructions from this manual, export
-user name and home directory now like this:
-
-    export USERNAME=nominatim
-    export USERHOME=/srv/nominatim
-
-**Never, ever run the installation as a root user.** You have been warned.
-
-Make sure that system servers can read from the home directory:
-
-    chmod a+x $USERHOME
-
-Setting up PostgreSQL
----------------------
-
-Tune the postgresql configuration, which is located in 
-`/etc/postgresql/9.5/main/postgresql.conf`. See section *Postgres Tuning* in
-[the installation page](Installation.md) for the parameters to change.
-
-Restart the postgresql service after updating this config file.
-
-    sudo systemctl restart postgresql
-
-
-Finally, we need to add two postgres users: one for the user that does
-the import and another for the webserver which should access the database
-for reading only:
-
-
-    sudo -u postgres createuser -s $USERNAME
-    sudo -u postgres createuser www-data
-
-
-Setting up the Apache Webserver
--------------------------------
-
-You need to create an alias to the website directory in your apache
-configuration. Add a separate nominatim configuration to your webserver:
-
-```
-sudo tee /etc/apache2/conf-available/nominatim.conf << EOFAPACHECONF
-<Directory "$USERHOME/Nominatim/build/website">
-  Options FollowSymLinks MultiViews
-  AddType text/html   .php
-  DirectoryIndex search.php
-  Require all granted
-</Directory>
-
-Alias /nominatim $USERHOME/Nominatim/build/website
-EOFAPACHECONF
-```
-
-
-
-
-Then enable the configuration and restart apache
-
-
-    sudo a2enconf nominatim
-    sudo systemctl restart apache2
-
-
-Installing Nominatim
-====================
-
-Building and Configuration
---------------------------
-
-Get the source code from Github and change into the source directory
-
-
-
-    cd $USERHOME
-    git clone --recursive git://github.com/openstreetmap/Nominatim.git
-    cd Nominatim
-
-
-
-
-
-When installing the latest source from github, you also need to
-download the country grid:
-
-
-    wget -O data/country_osm_grid.sql.gz http://www.nominatim.org/data/country_grid.sql.gz
-
-
-The code must be built in a separate directory. Create this directory,
-then configure and build Nominatim in there:
-
-    mkdir build
-    cd build
-    cmake $USERHOME/Nominatim
-    make
-
-You need to create a minimal configuration file that tells nominatim
-where it is located on the webserver:
-
-```
-tee settings/local.php << EOF
-<?php
- @define('CONST_Website_BaseURL', '/nominatim/');
-EOF
-```
-
-
-Nominatim is now ready to use. Continue with
-[importing a database from OSM data](Import-and-Update.md).

docs/admin/Faq.md

@@ -1,12 +1,8 @@
-Frequently Asked Questions
+# Running Your Own Instance
-==========================
-
-Running Your Own Instance
--------------------------
 
 ### Can I import only a few countries and also keep them up to date?
 
-You should use the extracts and updates from http://download.geofabrik.de.
+You should use the extracts and updates from https://download.geofabrik.de.
 For the intial import, download the countries you need and merge them.
 See [OSM Help](https://help.openstreetmap.org/questions/48843/merging-two-or-more-geographical-areas-to-import-two-or-more-osm-files-in-nominatim)
 for examples how to do that. Use the resulting single osm file when
@@ -25,8 +21,7 @@ for a script that runs the updates using osmosis.
 Make sure there are no spaces at the beginning of your `settings/local.php` file.
 
 
-Installation
+# Installation
-------------
 
 ### I accidentally killed the import process after it has been running for many hours. Can it be resumed?
 
@@ -34,13 +29,16 @@ It is possible if the import already got to the indexing stage.
 Check the last line of output that was logged before the process
 was killed. If it looks like this:
 
-   Done 844 in 13 @ 64.923080 per second - Rank 26 ETA (seconds): 7.886255
+
+    Done 844 in 13 @ 64.923080 per second - Rank 26 ETA (seconds): 7.886255
 
 then you can resume with the following command:
 
-   ./utils/setup.php --index --create-search-indices --create-country-names
+```sh
+./utils/setup.php --index --create-search-indices --create-country-names
+```
 
-If the reported rank is 26 or higher, you can also safely add `-index-noanalyse`.
+If the reported rank is 26 or higher, you can also safely add `--index-noanalyse`.
 
 
 ### When running the setup.php script I get a warning:
@@ -69,8 +67,9 @@ something like this:
 
 Or
 
-   echo "date.timezone = 'America/Denver'" > /etc/php.d/timezone.ini
+```
-
+echo "date.timezone = 'America/Denver'" > /etc/php.d/timezone.ini
+```
 
 ### When running the import I get a version mismatch:
     `COPY_END for place failed: ERROR: incompatible library "/opt/Nominatim/module/nominatim.so": version mismatch`
@@ -98,7 +97,7 @@ to get the full error message.
 On CentOS v7 the PostgreSQL server is started with `systemd`.
 Check if `/usr/lib/systemd/system/httpd.service` contains a line `PrivateTmp=true`.
 If so then Apache cannot see the `/tmp/.s.PGSQL.5432` file. It's a good security feature,
-so use the [[#PostgreSQL_UNIX_Socket_Location_on_CentOS|preferred solution]] 
+so use the [preferred solution](../appendix/Install-on-Centos-7/#adding-selinux-security-settings).
 
 However, you can solve this the quick and dirty way by commenting out that line and then run
 
@@ -120,7 +119,7 @@ have the [http://pear.php.net/package/DB/ Pear module 'DB'] installed.
 
     sudo pear install DB
 
-### I forgot to delete the flatnodes file before starting an import
+### I forgot to delete the flatnodes file before starting an import.
 
 That's fine. For each import the flatnodes file get overwritten.
 See https://help.openstreetmap.org/questions/52419/nominatim-flatnode-storage

docs/admin/Import-and-Update.md

@@ -1,16 +1,23 @@
-Importing a new database
-========================
-
 The following instructions explain how to create a Nominatim database
 from an OSM planet file and how to keep the database up to date. It
 is assumed that you have already successfully installed the Nominatim
 software itself, if not return to the [installation page](Installation.md).
 
-Configuration setup in settings/local.php
+# Configuration setup in settings/local.php
------------------------------------------
+
+The Nominatim server can be customized via the file `settings/local.php`
+in the build directory. Note that this is a PHP file, so it must always
+start like this:
+
+    <?php
+
+without any leading spaces.
 
 There are lots of configuration settings you can tweak. Have a look
-at `settings/settings.php` for a full list. Most should have a sensible default.
+at `settings/default.php` for a full list. Most should have a sensible default.
+
+### Flatnode files
+
 If you plan to import a large dataset (e.g. Europe, North America, planet),
 you should also enable flatnode storage of node locations. With this
 setting enabled, node coordinates are stored in a simple file instead
@@ -20,12 +27,11 @@ Add to your `settings/local.php`:
     @define('CONST_Osm2pgsql_Flatnode_File', '/path/to/flatnode.file');
 
 Replace the second part with a suitable path on your system and make sure
-the directory exists. There should be at least 35GB of free space.
+the directory exists. There should be at least 40GB of free space.
 
-Downloading additional data
+# Downloading additional data
----------------------------
 
-### Wikipedia rankings
+## Wikipedia rankings
 
 Wikipedia can be used as an optional auxiliary data source to help indicate
 the importance of osm features. Nominatim will work without this information
@@ -33,8 +39,8 @@ but it will improve the quality of the results if this is installed.
 This data is available as a binary download:
 
     cd $NOMINATIM_SOURCE_DIR/data
-    wget http://www.nominatim.org/data/wikipedia_article.sql.bin
+    wget https://www.nominatim.org/data/wikipedia_article.sql.bin
-    wget http://www.nominatim.org/data/wikipedia_redirect.sql.bin
+    wget https://www.nominatim.org/data/wikipedia_redirect.sql.bin
 
 Combined the 2 files are around 1.5GB and add around 30GB to the install
 size of nominatim. They also increase the install time by an hour or so.
@@ -43,45 +49,58 @@ size of nominatim. They also increase the install time by an hour or so.
 the initial import of the data if you want the rankings applied to the
 loaded data.
 
-### UK postcodes
+## UK postcodes
 
 Nominatim can use postcodes from an external source to improve searches that involve a UK postcode. This data can be optionally downloaded: 
 
     cd $NOMINATIM_SOURCE_DIR/data
-    wget http://www.nominatim.org/data/gb_postcode_data.sql.gz
+    wget https://www.nominatim.org/data/gb_postcode_data.sql.gz
 
 
-Initial import of the data
+# Initial import of the data
---------------------------
 
 **Important:** first try the import with a small excerpt, for example from
-[Geofabrik](http://download.geofabrik.de).
+[Geofabrik](https://download.geofabrik.de).
 
 Download the data to import and load the data with the following command:
 
-    ./utils/setup.php --osm-file <your data file> --all [--osm2pgsql-cache 28000] 2>&1 | tee setup.log
+```sh
+./utils/setup.php --osm-file <data file> --all [--osm2pgsql-cache 28000] 2>&1 | tee setup.log
+```
 
 The `--osm2pgsql-cache` parameter is optional but strongly recommended for
 planet imports. It sets the node cache size for the osm2pgsql import part
-(see `-C` parameter in osm2pgsql help). 28GB are recommended for a full planet
+(see `-C` parameter in osm2pgsql help). As a rule of thumb, this should be
-import, for excerpts you can use less. Adapt to your available RAM to
+about the same size as the file you are importing but never more than
-avoid swapping, never give more than 2/3 of RAM to osm2pgsql.
+2/3 of RAM available. If your machine starts swapping reduce the size.
 
+Computing word frequency for search terms can improve the performance of
+forward geocoding in particular under high load as it helps Postgres' query
+planner to make the right decisions. To recompute word counts run:
 
-Loading additional datasets
+```sh
----------------------------
+./utils/update.php --recompute-word-counts
+```
 
-The following commands will create additional entries for POI searches:
+This will take a couple of hours for a full planet installation. You can
+also defer that step to a later point in time when you realise that
+performance becomes an issue. Just make sure that updates are stopped before
+running this function.
+
+If you want to be able to search for places by their type through
+[special key phrases](https://wiki.openstreetmap.org/wiki/Nominatim/Special_Phrases)
+you also need to enable these key phrases like this:
 
     ./utils/specialphrases.php --wiki-import > specialphrases.sql
     psql -d nominatim -f specialphrases.sql
 
+Note that this command downloads the phrases from the wiki link above.
 
-Installing Tiger housenumber data for the US
+
-============================================
+# Installing Tiger housenumber data for the US
 
 Nominatim is able to use the official TIGER address set to complement the
-OSM housenumber data in the US. You can add TIGER data to your own Nominatim
+OSM house number data in the US. You can add TIGER data to your own Nominatim
 instance by following these steps:
 
   1. Install the GDAL library and python bindings and the unzip tool
@@ -89,48 +108,62 @@ instance by following these steps:
        * Ubuntu: `sudo apt-get install python-gdal unzip`
        * CentOS: `sudo yum install gdal-python unzip`
 
-  2. Get the TIGER 2015 data. You will need the EDGES files
+  2. Get preprocessed TIGER 2015 data and unpack it into the
+     data directory in your Nominatim sources:
+
+        cd Nominatim/data
+        wget https://nominatim.org/data/tiger2017-nominatim-preprocessed.tar.gz
+        tar xf tiger2017-nominatim-preprocessed.tar.gz
+
+  3. Import the data into your Nominatim database: 
+
+        ./utils/setup.php --import-tiger-data
+
+  4. Enable use of the Tiger data in your `settings/local.php` by adding:
+
+         @define('CONST_Use_US_Tiger_Data', true);
+
+  5. Apply the new settings:
+
+```sh
+    ./utils/setup.php --create-functions --enable-diff-updates --create-partition-functions
+```
+
+The entire US adds about 10GB to your database.
+
+You can also process the data from the original TIGER data to create the
+SQL files, Nominatim needs for the import:
+
+  1. Get the TIGER 2017 data. You will need the EDGES files
      (3,234 zip files, 11GB total). Choose one of the two sources:
 
-         wget -r ftp://ftp2.census.gov/geo/tiger/TIGER2015/EDGES/
+         wget -r ftp://ftp2.census.gov/geo/tiger/TIGER2017/EDGES/
-         wget -r ftp://mirror1.shellbot.com/census/geo/tiger/TIGER2015/EDGES/
+         wget -r ftp://mirror1.shellbot.com/census/geo/tiger/TIGER2017/EDGES/
 
      The first one is the original source, the second a considerably faster
      mirror.
 
-  3. Convert the data into SQL statements (stored in data/tiger): 
+  2. Convert the data into SQL statements: 
 
          ./utils/imports.php --parse-tiger <tiger edge data directory>
 
-  4. Import the data into your Nominatim database: 
+Be warned that this can take quite a long time. After this process is finished,
+the same preprocessed files as above are available in `data/tiger`.
 
-         ./utils/setup.php --import-tiger-data
+# Updates
-
-  5. Enable use of the Tiger data in your `settings/local.php` by adding:
-
-         @define('CONST_Use_US_Tiger_Data', true);
-
-  6. Apply the new settings:
-
-         ./utils/setup.php --create-functions --enable-diff-updates --create-partition-functions
-
-Be warned that the import can take a very long time, especially if you
-import all of the US. The entire US adds about 10GB to your database.
-
-
-Updates
-=======
 
 There are many different possibilities to update your Nominatim database.
 The following section describes how to keep it up-to-date with Pyosmium.
 For a list of other methods see the output of `./utils/update.php --help`.
 
-Installing the newest version of Pyosmium
+### Installing the newest version of Pyosmium
------------------------------------------
 
-It is recommended to install Pyosmium via pip:
+It is recommended to install Pyosmium via pip. Run (as the same user who
+will later run the updates):
 
-    pip install --user osmium
+```sh
+pip install --user osmium
+```
 
 Nominatim needs a tool called `pyosmium-get-updates` that comes with
 Pyosmium. You need to tell Nominatim where to find it. Add the
@@ -141,8 +174,7 @@ following line to your `settings/local.php`:
 The path above is fine if you used the `--user` parameter with pip.
 Replace `user` with your user name.
 
-Setting up the update process
+### Setting up the update process
------------------------------
 
 Next the update needs to be initialised. By default Nominatim is configured
 to update using the global minutely diffs.
@@ -152,7 +184,7 @@ to `settings/local.php`. For example, to use the daily country extracts
 diffs for Ireland from geofabrik add the following:
 
     // base URL of the replication service
-    @define('CONST_Replication_Url', 'http://download.geofabrik.de/europe/ireland-and-northern-ireland-updates');
+    @define('CONST_Replication_Url', 'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates');
     // How often upstream publishes diffs
     @define('CONST_Replication_Update_Interval', '86400');
     // How long to sleep if no update found yet
@@ -168,8 +200,7 @@ what you expect.
 The --init-updates command needs to be rerun whenever the replication service
 is changed.
 
-Updating Nominatim
+### Updating Nominatim
-------------------
 
 The following command will keep your database constantly up to date:
 

docs/admin/Installation.md

@@ -1,20 +1,16 @@
-Nominatim installation
-======================
-
 This page contains generic installation instructions for Nominatim and its
 prerequisites. There are also step-by-step instructions available for
 the following operating systems:
 
-  * [Ubuntu 16.04](Install-on-Ubuntu-16.md)
+  * [Ubuntu 16.04](../appendix/Install-on-Ubuntu-16.md)
-  * [CentOS 7.2](Install-on-Centos-7.md)
+  * [CentOS 7.2](../appendix/Install-on-Centos-7.md)
 
 These OS-specific instructions can also be found in executable form
 in the `vagrant/` directory.
 
-Prerequisites
+# Prerequisites
--------------
 
-### Software
+## Software
 
 For compiling:
 
@@ -23,8 +19,7 @@ For compiling:
   * a recent C++ compiler
 
 Nominatim comes with its own version of osm2pgsql. See the
-[osm2pgsql README](../osm2pgsql/README.md) for additional dependencies
+osm2pgsql README for additional dependencies required for compiling osm2pgsql.
-required for compiling osm2pgsql.
 
 For running tests:
 
@@ -47,23 +42,22 @@ For running continuous updates:
 
   * [pyosmium](http://osmcode.org/pyosmium/)
 
-### Hardware
+## Hardware
 
 A minimum of 2GB of RAM is required or installation will fail. For a full
 planet import 32GB of RAM or more strongly are recommended.
 
-For a full planet install you will need about 500GB of hard disk space (as of
+For a full planet install you will need about 600GB of hard disk space (as of
-June 2016, take into account that the OSM database is growing fast). SSD disks
+January 2017, take into account that the OSM database is growing fast). SSD disks
 will help considerably to speed up import and queries.
 
 On a 6-core machine with 32GB RAM and SSDs the import of a full planet takes
 a bit more than 2 days. Without SSDs 7-8 days are more realistic.
 
 
-Setup of the server
+# Setup of the server
--------------------
 
-### PostgreSQL tuning
+## PostgreSQL tuning
 
 You might want to tune your PostgreSQL installation so that the later steps
 make best use of your hardware. You should tune the following parameters in
@@ -90,13 +84,13 @@ Don't forget to reenable them after the initial import or you risk database
 corruption. Autovacuum must not be switched off because it ensures that the
 tables are frequently analysed.
 
-### Webserver setup
+## Webserver setup
 
 The `website/` directory in the build directory contains the configured
 website. Include the directory into your webbrowser to serve php files
 from there.
 
-#### Configure for use with Apache
+### Configure for use with Apache
 
 Make sure your Apache configuration contains the required permissions for the
 directory and create an alias:
@@ -115,7 +109,7 @@ build directory.
 After making changes in the apache config you need to restart apache.
 The website should now be available on http://localhost/nominatim.
 
-#### Configure for use with Nginx
+### Configure for use with Nginx
 
 Use php-fpm as a deamon for serving PHP cgi. Install php-fpm together with nginx.
 
@@ -148,7 +142,7 @@ unix socket by adding the location definition to the default configuration.
     }
 
 Restart the nginx and php5-fpm services and the website should now be available
-on http://localhost/.
+at `http://localhost/`.
 
 
 Now continue with [importing the database](Import-and-Update.md).

docs/admin/Migration.md

@@ -0,0 +1,66 @@
+Database Migrations
+===================
+
+This page describes database migrations necessary to update existing databases
+to newer versions of Nominatim.
+
+SQL statements should be executed from the postgres commandline. Execute
+`psql nominiatim` to enter command line mode.
+
+# 3.0.0 -> 3.1.0
+
+### Postcode Table
+
+A new separate table for artificially computed postcode centroids was introduced.
+Migration to the new format is possible but **not recommended**.
+
+Create postcode table and indexes, running the following SQL statements:
+
+```sql
+CREATE TABLE location_postcode
+  (place_id BIGINT, parent_place_id BIGINT, rank_search SMALLINT,
+   rank_address SMALLINT, indexed_status SMALLINT, indexed_date TIMESTAMP,
+   country_code varchar(2), postcode TEXT,
+   geometry GEOMETRY(Geometry, 4326));
+CREATE INDEX idx_postcode_geometry ON location_postcode USING GIST (geometry);
+CREATE UNIQUE INDEX idx_postcode_id ON location_postcode USING BTREE (place_id);
+CREATE INDEX idx_postcode_postcode ON location_postcode USING BTREE (postcode);
+GRANT SELECT ON location_postcode TO "www-data";
+```
+
+Add postcode column to `location_area` tables with SQL statement:
+
+```sql
+ALTER TABLE location_area ADD COLUMN postcode TEXT;
+```
+
+Then reimport the functions:
+
+```sh
+./utils/setup.php --create-functions --enable-diff-updates --create-partition-functions
+```
+
+Create appropriate triggers with SQL:
+
+```sql
+CREATE TRIGGER location_postcode_before_update BEFORE UPDATE ON location_postcode
+    FOR EACH ROW EXECUTE PROCEDURE postcode_update();
+```
+
+Finally populate the postcode table (will take a while):
+
+```sh
+./utils/setup.php --calculate-postcodes --index --index-noanalyse
+```
+
+This will create a working database. You may also delete the old artificial
+postcodes now. Note that this may be expensive and is not absolutely necessary.
+The following SQL statement will remove them:
+
+```sql
+DELETE FROM place_addressline a USING placex p
+ WHERE a.address_place_id = p.place_id and p.osm_type = 'P';
+ALTER TABLE placex DISABLE TRIGGER USER;
+DELETE FROM placex WHERE osm_type = 'P';
+ALTER TABLE placex ENABLE TRIGGER USER;
+```

docs/index.md

@@ -0,0 +1 @@
+Nominatim (from the Latin, 'by name') is a tool to search OSM data by name and address and to generate synthetic addresses of OSM points (reverse geocoding).

docs/mkdocs.yml

@@ -0,0 +1,18 @@
+site_name: Nominatim Documentation
+theme: readthedocs
+docs_dir: ${CMAKE_CURRENT_BINARY_DIR}
+site_url: http://nominatim.org
+repo_url: https://github.com/openstreetmap/Nominatim
+pages:
+    - 'Introduction' : 'index.md'
+    - 'Administration Guide':
+        - 'Basic Installation': 'admin/Installation.md'
+        - 'Importing and Updating' : 'admin/Import-and-Update.md'
+        - 'Migration from older Versions' : 'admin/Migration.md'
+        - 'Troubleshooting' : 'admin/Faq.md'
+    - 'Appendix':
+        - 'Installation on CentOS 7' : 'appendix/Install-on-Centos-7.md'
+        - 'Installation on Ubuntu 16' : 'appendix/Install-on-Ubuntu-16.md'
+markdown_extensions:
+    - codehilite:
+        use_pygments: False

lib/NearPoint.php

@@ -1,157 +0,0 @@
-<?php
-
-namespace Nominatim;
-
-/**
- * A geographic point with a search radius.
- */
-class NearPoint
-{
-    private $fLat;
-    private $fLon;
-    private $fRadius;
-
-    private $sSQL;
-
-
-    public function __construct($lat, $lon, $radius = 0.1)
-    {
-        $this->fLat = (float)$lat;
-        $this->fLon = (float)$lon;
-        $this->fRadius = (float)$radius;
-        $this->sSQL = 'ST_SetSRID(ST_Point('.$this->fLon.','.$this->fLat.'),4326)';
-    }
-
-    public function lat()
-    {
-        return $this->fLat;
-    }
-
-    public function lon()
-    {
-        return $this->fLon;
-    }
-
-    public function radius()
-    {
-        return $this->fRadius;
-    }
-
-    public function distanceSQL($sObj)
-    {
-        return 'ST_Distance('.$this->sSQL.", $sObj)";
-    }
-
-    public function withinSQL($sObj)
-    {
-        return sprintf('ST_DWithin(%s, %s, %F)', $sObj, $this->sSQL, $this->fRadius);
-    }
-
-    /**
-     * Check that the coordinates are valid WSG84 coordinates.
-     *
-     * @return bool True if the coordinates are correctly bounded.
-     */
-    public function isValid()
-    {
-        return ($this->fLat <= 90.1
-                && $this->fLat >= -90.1
-                && $this->fLon <= 180.1
-                && $this->fLon >= -180.1);
-    }
-
-    /**
-     * Extract a coordinate point from a query string.
-     *
-     * If a coordinate is found an array of a new NearPoint and the
-     * remaining query is returned or false otherwise.
-     *
-     * @param string $sQuery Query to scan.
-     *
-     * @return array|false If a coordinate was found, an array with
-     *                     `pt` as the NearPoint coordinates and `query`
-     *                      with the remaining query string. False otherwiese.
-     */
-    public static function extractFromQuery($sQuery)
-    {
-        // Do we have anything that looks like a lat/lon pair?
-        // returns array(lat,lon,query_with_lat_lon_removed)
-        // or null
-        $sFound    = null;
-        $fQueryLat = null;
-        $fQueryLon = null;
-
-        if (preg_match('/\\b([NS])[ ]+([0-9]+[0-9.]*)[° ]+([0-9.]+)?[′\']*[, ]+([EW])[ ]+([0-9]+)[° ]+([0-9]+[0-9.]*)[′\']*?\\b/', $sQuery, $aData)) {
-            /*              1         2                   3                  4         5            6
-             * degrees decimal minutes
-             * N 40 26.767, W 79 58.933
-             * N 40°26.767′, W 79°58.933′
-             */
-            $sFound    = $aData[0];
-            $fQueryLat = ($aData[1]=='N'?1:-1) * ($aData[2] + $aData[3]/60);
-            $fQueryLon = ($aData[4]=='E'?1:-1) * ($aData[5] + $aData[6]/60);
-        } elseif (preg_match('/\\b([0-9]+)[° ]+([0-9]+[0-9.]*)?[′\']*[ ]+([NS])[, ]+([0-9]+)[° ]+([0-9]+[0-9.]*)?[′\' ]+([EW])\\b/', $sQuery, $aData)) {
-            /*                    1             2                      3          4            5                    6
-             * degrees decimal minutes
-             * 40 26.767 N, 79 58.933 W
-             * 40° 26.767′ N 79° 58.933′ W
-             */
-            $sFound    = $aData[0];
-            $fQueryLat = ($aData[3]=='N'?1:-1) * ($aData[1] + $aData[2]/60);
-            $fQueryLon = ($aData[6]=='E'?1:-1) * ($aData[4] + $aData[5]/60);
-        } elseif (preg_match('/\\b([NS])[ ]([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″"]*[, ]+([EW])[ ]([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″"]*\\b/', $sQuery, $aData)) {
-            /*                    1        2            3            4                5        6            7            8
-             * degrees decimal seconds
-             * N 40 26 46 W 79 58 56
-             * N 40° 26′ 46″, W 79° 58′ 56″
-             */
-            $sFound    = $aData[0];
-            $fQueryLat = ($aData[1]=='N'?1:-1) * ($aData[2] + $aData[3]/60 + $aData[4]/3600);
-            $fQueryLon = ($aData[5]=='E'?1:-1) * ($aData[6] + $aData[7]/60 + $aData[8]/3600);
-        } elseif (preg_match('/\\b([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″" ]+([NS])[, ]+([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″" ]+([EW])\\b/', $sQuery, $aData)) {
-            /*                    1            2            3            4          5            6            7            8
-             * degrees decimal seconds
-             * 40 26 46 N 79 58 56 W
-             * 40° 26′ 46″ N, 79° 58′ 56″ W
-             */
-            $sFound    = $aData[0];
-            $fQueryLat = ($aData[4]=='N'?1:-1) * ($aData[1] + $aData[2]/60 + $aData[3]/3600);
-            $fQueryLon = ($aData[8]=='E'?1:-1) * ($aData[5] + $aData[6]/60 + $aData[7]/3600);
-        } elseif (preg_match('/\\b([NS])[ ]([0-9]+[0-9]*\\.[0-9]+)[°]*[, ]+([EW])[ ]([0-9]+[0-9]*\\.[0-9]+)[°]*\\b/', $sQuery, $aData)) {
-            /*                    1        2                               3        4
-             * degrees decimal
-             * N 40.446° W 79.982°
-             */
-            $sFound    = $aData[0];
-            $fQueryLat = ($aData[1]=='N'?1:-1) * ($aData[2]);
-            $fQueryLon = ($aData[3]=='E'?1:-1) * ($aData[4]);
-        } elseif (preg_match('/\\b([0-9]+[0-9]*\\.[0-9]+)[° ]+([NS])[, ]+([0-9]+[0-9]*\\.[0-9]+)[° ]+([EW])\\b/', $sQuery, $aData)) {
-            /*                    1                           2          3                           4
-             * degrees decimal
-             * 40.446° N 79.982° W
-             */
-            $sFound    = $aData[0];
-            $fQueryLat = ($aData[2]=='N'?1:-1) * ($aData[1]);
-            $fQueryLon = ($aData[4]=='E'?1:-1) * ($aData[3]);
-        } elseif (preg_match('/(\\[|^|\\b)(-?[0-9]+[0-9]*\\.[0-9]+)[, ]+(-?[0-9]+[0-9]*\\.[0-9]+)(\\]|$|\\b)/', $sQuery, $aData)) {
-            /*                 1          2                             3                        4
-             * degrees decimal
-             * 12.34, 56.78
-             * [12.456,-78.90]
-             */
-            $sFound    = $aData[0];
-            $fQueryLat = $aData[2];
-            $fQueryLon = $aData[3];
-        } else {
-            return false;
-        }
-
-        $oPt = new NearPoint($fQueryLat, $fQueryLon);
-
-        if (!$oPt->isValid()) return false;
-
-        $sQuery = trim(str_replace($sFound, ' ', $sQuery));
-
-        return array('pt' => $oPt, 'query' => $sQuery);
-    }
-}

lib/ParameterParser.php

@@ -82,8 +82,8 @@ class ParameterParser
 
     public function getPreferredLanguages($sFallback = null)
     {
-        if ($sFallback === null && isset($_SERVER["HTTP_ACCEPT_LANGUAGE"])) {
+        if ($sFallback === null && isset($_SERVER['HTTP_ACCEPT_LANGUAGE'])) {
-            $sFallback = $_SERVER["HTTP_ACCEPT_LANGUAGE"];
+            $sFallback = $_SERVER['HTTP_ACCEPT_LANGUAGE'];
         }
 
         $aLanguages = array();

lib/Phrase.php

@@ -0,0 +1,116 @@
+<?php
+
+namespace Nominatim;
+
+/**
+ * Segment of a query string.
+ *
+ * The parts of a query strings are usually separated by commas.
+ */
+class Phrase
+{
+    const MAX_DEPTH = 7;
+
+    // Complete phrase as a string.
+    private $sPhrase;
+    // Element type for structured searches.
+    private $sPhraseType;
+    // Space-separated words of the phrase.
+    private $aWords;
+    // Possible segmentations of the phrase.
+    private $aWordSets;
+
+
+    public function __construct($sPhrase, $sPhraseType)
+    {
+        $this->sPhrase = trim($sPhrase);
+        $this->sPhraseType = $sPhraseType;
+        $this->aWords = explode(' ', $this->sPhrase);
+        $this->aWordSets = $this->createWordSets($this->aWords, 0);
+    }
+
+    /**
+     * Return the element type of the phrase.
+     *
+     * @return string Pharse type if the phrase comes from a structured query
+     *                or empty string otherwise.
+     */
+    public function getPhraseType()
+    {
+        return $this->sPhraseType;
+    }
+
+    /**
+     * Return the array of possible segmentations of the phrase.
+     *
+     * @return string[][] Array of segmentations, each consisting of an
+     *                    array of terms.
+     */
+    public function getWordSets()
+    {
+        return $this->aWordSets;
+    }
+
+    /**
+     * Add the tokens from this phrase to the given list of tokens.
+     *
+     * @param string[] $aTokens List of tokens to append.
+     *
+     * @return void
+     */
+    public function addTokens(&$aTokens)
+    {
+        foreach ($this->aWordSets as $aSet) {
+            foreach ($aSet as $sWord) {
+                $aTokens[' '.$sWord] = ' '.$sWord;
+                $aTokens[$sWord] = $sWord;
+            }
+        }
+    }
+
+    /**
+     * Invert the set of possible segmentations.
+     *
+     * @return void
+     */
+    public function invertWordSets()
+    {
+        $this->aWordSets = $this->createInverseWordSets($this->aWords, 0);
+    }
+
+    private function createWordSets($aWords, $iDepth)
+    {
+        $aResult = array(array(join(' ', $aWords)));
+        $sFirstToken = '';
+        if ($iDepth < Phrase::MAX_DEPTH) {
+            while (sizeof($aWords) > 1) {
+                $sWord = array_shift($aWords);
+                $sFirstToken .= ($sFirstToken?' ':'').$sWord;
+                $aRest = $this->createWordSets($aWords, $iDepth + 1);
+                foreach ($aRest as $aSet) {
+                    $aResult[] = array_merge(array($sFirstToken), $aSet);
+                }
+            }
+        }
+
+        return $aResult;
+    }
+
+    private function createInverseWordSets($aWords, $iDepth)
+    {
+        $aResult = array(array(join(' ', $aWords)));
+        $sFirstToken = '';
+        if ($iDepth < Phrase::MAX_DEPTH) {
+            while (sizeof($aWords) > 1) {
+                $sWord = array_pop($aWords);
+                $sFirstToken = $sWord.($sFirstToken?' ':'').$sFirstToken;
+                $aRest = $this->createInverseWordSets($aWords, $iDepth + 1);
+                foreach ($aRest as $aSet) {
+                    $aResult[] = array_merge(array($sFirstToken), $aSet);
+                }
+            }
+        }
+
+        return $aResult;
+    }
+}

lib/PlaceLookup.php

@@ -2,11 +2,13 @@
 
 namespace Nominatim;
 
+require_once(CONST_BasePath.'/lib/Result.php');
+
 class PlaceLookup
 {
     protected $oDB;
 
-    protected $aLangPrefOrder = array();
+    protected $aLangPrefOrderSql = "''";
 
     protected $bAddressDetails = false;
     protected $bExtraTags = false;
@@ -19,15 +21,110 @@ class PlaceLookup
     protected $bIncludePolygonAsSVG = false;
     protected $fPolygonSimplificationThreshold = 0.0;
 
+    protected $sAnchorSql = null;
+    protected $sAddressRankListSql = null;
+    protected $sAllowedTypesSQLList = null;
+    protected $bDeDupe = true;
+
 
     public function __construct(&$oDB)
     {
         $this->oDB =& $oDB;
     }
 
+    public function doDeDupe()
+    {
+        return $this->bDeDupe;
+    }
+
+    public function setIncludePolygonAsPoints($b = true)
+    {
+        $this->bIncludePolygonAsPoints = $b;
+    }
+
+    public function loadParamArray($oParams, $sGeomType = null)
+    {
+        $aLangs = $oParams->getPreferredLanguages();
+        $this->aLangPrefOrderSql =
+            'ARRAY['.join(',', array_map('getDBQuoted', $aLangs)).']';
+
+        $this->bAddressDetails = $oParams->getBool('addressdetails', true);
+        $this->bExtraTags = $oParams->getBool('extratags', false);
+        $this->bNameDetails = $oParams->getBool('namedetails', false);
+
+        $this->bDeDupe = $oParams->getBool('dedupe', $this->bDeDupe);
+
+        if ($sGeomType === null || $sGeomType == 'text') {
+            $this->bIncludePolygonAsText = $oParams->getBool('polygon_text');
+        }
+        if ($sGeomType === null || $sGeomType == 'geojson') {
+            $this->bIncludePolygonAsGeoJSON = $oParams->getBool('polygon_geojson');
+        }
+        if ($sGeomType === null || $sGeomType == 'kml') {
+            $this->bIncludePolygonAsKML = $oParams->getBool('polygon_kml');
+        }
+        if ($sGeomType === null || $sGeomType == 'svg') {
+            $this->bIncludePolygonAsSVG = $oParams->getBool('polygon_svg');
+        }
+        $this->fPolygonSimplificationThreshold
+            = $oParams->getFloat('polygon_threshold', 0.0);
+
+        $iWantedTypes =
+            ($this->bIncludePolygonAsText ? 1 : 0) +
+            ($this->bIncludePolygonAsGeoJSON ? 1 : 0) +
+            ($this->bIncludePolygonAsKML ? 1 : 0) +
+            ($this->bIncludePolygonAsSVG ? 1 : 0);
+        if ($iWantedTypes > CONST_PolygonOutput_MaximumTypes) {
+            if (CONST_PolygonOutput_MaximumTypes) {
+                userError('Select only '.CONST_PolygonOutput_MaximumTypes.' polgyon output option');
+            } else {
+                userError('Polygon output is disabled');
+            }
+        }
+    }
+
+    public function getMoreUrlParams()
+    {
+        $aParams = array();
+
+        if ($this->bAddressDetails) $aParams['addressdetails'] = '1';
+        if ($this->bExtraTags) $aParams['extratags'] = '1';
+        if ($this->bNameDetails) $aParams['namedetails'] = '1';
+
+        if ($this->bIncludePolygonAsPoints) $aParams['polygon'] = '1';
+        if ($this->bIncludePolygonAsText) $aParams['polygon_text'] = '1';
+        if ($this->bIncludePolygonAsGeoJSON) $aParams['polygon_geojson'] = '1';
+        if ($this->bIncludePolygonAsKML) $aParams['polygon_kml'] = '1';
+        if ($this->bIncludePolygonAsSVG) $aParams['polygon_svg'] = '1';
+
+        if ($this->fPolygonSimplificationThreshold > 0.0) {
+            $aParams['polygon_threshold'] = $this->fPolygonSimplificationThreshold;
+        }
+
+        if (!$this->bDeDupe) $aParams['dedupe'] = '0';
+
+        return $aParams;
+    }
+
+    public function setAnchorSql($sPoint)
+    {
+        $this->sAnchorSql = $sPoint;
+    }
+
+    public function setAddressRankList($aList)
+    {
+        $this->sAddressRankListSql = '('.join(',', $aList).')';
+    }
+
+    public function setAllowedTypesSQLList($sSql)
+    {
+        $this->sAllowedTypesSQLList = $sSql;
+    }
+
     public function setLanguagePreference($aLangPrefOrder)
     {
-        $this->aLangPrefOrder = $aLangPrefOrder;
+        $this->aLangPrefOrderSql =
+            'ARRAY['.join(',', array_map('getDBQuoted', $aLangPrefOrder)).']';
     }
 
     public function setIncludeAddressDetails($bAddressDetails = true)
@@ -35,180 +132,356 @@ class PlaceLookup
         $this->bAddressDetails = $bAddressDetails;
     }
 
-    public function setIncludeExtraTags($bExtraTags = false)
+    private function addressImportanceSql($sGeometry, $sPlaceId)
     {
-        $this->bExtraTags = $bExtraTags;
+        if ($this->sAnchorSql) {
+            $sSQL = 'ST_Distance('.$this->sAnchorSql.','.$sGeometry.')';
+        } else {
+            $sSQL = '(SELECT max(ai_p.importance * (ai_p.rank_address + 2))';
+            $sSQL .= '   FROM place_addressline ai_s, placex ai_p';
+            $sSQL .= '   WHERE ai_s.place_id = '.$sPlaceId;
+            $sSQL .= '     AND ai_p.place_id = ai_s.address_place_id ';
+            $sSQL .= '     AND ai_s.isaddress ';
+            $sSQL .= '     AND ai_p.importance is not null)';
+        }
+
+        return $sSQL.' AS addressimportance,';
     }
 
-    public function setIncludeNameDetails($bNameDetails = false)
+    private function langAddressSql($sHousenumber)
     {
-        $this->bNameDetails = $bNameDetails;
+        return 'get_address_by_language(place_id,'.$sHousenumber.','.$this->aLangPrefOrderSql.') AS langaddress,';
-    }
-
-
-    public function setIncludePolygonAsPoints($b = true)
-    {
-        $this->bIncludePolygonAsPoints = $b;
-    }
-
-    public function getIncludePolygonAsPoints()
-    {
-        return $this->bIncludePolygonAsPoints;
-    }
-
-    public function setIncludePolygonAsText($b = true)
-    {
-        $this->bIncludePolygonAsText = $b;
-    }
-
-    public function getIncludePolygonAsText()
-    {
-        return $this->bIncludePolygonAsText;
-    }
-
-    public function setIncludePolygonAsGeoJSON($b = true)
-    {
-        $this->bIncludePolygonAsGeoJSON = $b;
-    }
-
-    public function setIncludePolygonAsKML($b = true)
-    {
-        $this->bIncludePolygonAsKML = $b;
-    }
-
-    public function setIncludePolygonAsSVG($b = true)
-    {
-        $this->bIncludePolygonAsSVG = $b;
-    }
-
-    public function setPolygonSimplificationThreshold($f)
-    {
-        $this->fPolygonSimplificationThreshold = $f;
     }
 
     public function lookupOSMID($sType, $iID)
     {
-        $sSQL = "select place_id from placex where osm_type = '".pg_escape_string($sType)."' and osm_id = ".(int)$iID." order by type = 'postcode' asc";
+        $sSQL = "select place_id from placex where osm_type = '".$sType."' and osm_id = ".$iID;
         $iPlaceID = chksql($this->oDB->getOne($sSQL));
 
-        return $this->lookup((int)$iPlaceID);
+        if (!$iPlaceID) {
+            return null;
+        }
+
+        $aResults = $this->lookup(array($iPlaceID => new Result($iPlaceID)));
+
+        return sizeof($aResults) ? reset($aResults) : null;
     }
 
-    public function lookup($iPlaceID, $sType = '', $fInterpolFraction = 0.0)
+    public function lookup($aResults, $iMinRank = 0, $iMaxRank = 30)
     {
-        if (!$iPlaceID) return null;
+        if (!sizeof($aResults)) {
+            return array();
+        }
+        $aSubSelects = array();
 
-        $sLanguagePrefArraySQL = "ARRAY[".join(',', array_map("getDBQuoted", $this->aLangPrefOrder))."]";
+        $sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_PLACEX);
-        $bIsTiger = CONST_Use_US_Tiger_Data && $sType == 'tiger';
+        if (CONST_Debug) var_dump('PLACEX', $sPlaceIDs);
-        $bIsInterpolation = $sType == 'interpolation';
+        if ($sPlaceIDs) {
+            $sSQL  = 'SELECT ';
+            $sSQL .= '    osm_type,';
+            $sSQL .= '    osm_id,';
+            $sSQL .= '    class,';
+            $sSQL .= '    type,';
+            $sSQL .= '    admin_level,';
+            $sSQL .= '    rank_search,';
+            $sSQL .= '    rank_address,';
+            $sSQL .= '    min(place_id) AS place_id,';
+            $sSQL .= '    min(parent_place_id) AS parent_place_id,';
+            $sSQL .= '    -1 as housenumber,';
+            $sSQL .= '    country_code,';
+            $sSQL .= $this->langAddressSql('-1');
+            $sSQL .= '    get_name_by_language(name,'.$this->aLangPrefOrderSql.') AS placename,';
+            $sSQL .= "    get_name_by_language(name, ARRAY['ref']) AS ref,";
+            if ($this->bExtraTags) {
+                $sSQL .= 'hstore_to_json(extratags)::text AS extra,';
+            }
+            if ($this->bNameDetails) {
+                $sSQL .= 'hstore_to_json(name)::text AS names,';
+            }
+            $sSQL .= '    avg(ST_X(centroid)) AS lon, ';
+            $sSQL .= '    avg(ST_Y(centroid)) AS lat, ';
+            $sSQL .= '    COALESCE(importance,0.75-(rank_search::float/40)) AS importance, ';
+            $sSQL .= $this->addressImportanceSql(
+                'ST_Collect(centroid)',
+                'min(CASE WHEN placex.rank_search < 28 THEN placex.place_id ELSE placex.parent_place_id END)'
+            );
+            $sSQL .= "    (extratags->'place') AS extra_place ";
+            $sSQL .= ' FROM placex';
+            $sSQL .= " WHERE place_id in ($sPlaceIDs) ";
+            $sSQL .= '   AND (';
+            $sSQL .= "        placex.rank_address between $iMinRank and $iMaxRank ";
+            if (14 >= $iMinRank && 14 <= $iMaxRank) {
+                $sSQL .= "    OR (extratags->'place') = 'city'";
+            }
+            if ($this->sAddressRankListSql) {
+                $sSQL .= '    OR placex.rank_address in '.$this->sAddressRankListSql;
+            }
+            $sSQL .= '       ) ';
+            if ($this->sAllowedTypesSQLList) {
+                $sSQL .= 'AND placex.class in '.$this->sAllowedTypesSQLList;
+            }
+            $sSQL .= '    AND linked_place_id is null ';
+            $sSQL .= ' GROUP BY ';
+            $sSQL .= '     osm_type, ';
+            $sSQL .= '     osm_id, ';
+            $sSQL .= '     class, ';
+            $sSQL .= '     type, ';
+            $sSQL .= '     admin_level, ';
+            $sSQL .= '     rank_search, ';
+            $sSQL .= '     rank_address, ';
+            $sSQL .= '     housenumber,';
+            $sSQL .= '     country_code, ';
+            $sSQL .= '     importance, ';
+            if (!$this->bDeDupe) $sSQL .= 'place_id,';
+            $sSQL .= '     langaddress, ';
+            $sSQL .= '     placename, ';
+            $sSQL .= '     ref, ';
+            if ($this->bExtraTags) $sSQL .= 'extratags, ';
+            if ($this->bNameDetails) $sSQL .= 'name, ';
+            $sSQL .= "     extratags->'place' ";
 
-        if ($bIsTiger) {
+            $aSubSelects[] = $sSQL;
-            $sSQL = "select place_id,partition, 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, housenumber, postcode,";
-            $sSQL .= " 'us' as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
-            $sSQL .= " coalesce(null,0.75-(30::float/40)) as importance, null as indexed_status, null as indexed_date, null as wikipedia, 'us' as country_code, ";
-            $sSQL .= " get_address_by_language(place_id, housenumber, $sLanguagePrefArraySQL) as langaddress,";
-            $sSQL .= " null as placename,";
-            $sSQL .= " null as ref,";
-            if ($this->bExtraTags) $sSQL .= " null as extra,";
-            if ($this->bNameDetails) $sSQL .= " null as names,";
-            $sSQL .= " ST_X(point) as lon, ST_Y(point) as lat from (select *, ST_LineInterpolatePoint(linegeo, (housenumber-startnumber::float)/(endnumber-startnumber)::float) as point from ";
-            $sSQL .= " (select *, ";
-            $sSQL .= " CASE WHEN interpolationtype='odd' THEN floor((".$fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2+1";
-            $sSQL .= " WHEN interpolationtype='even' THEN ((".$fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2";
-            $sSQL .= " WHEN interpolationtype='all' THEN (".$fInterpolFraction."*(endnumber-startnumber)+startnumber)::int";
-            $sSQL .= " END as housenumber";
-            $sSQL .= " from location_property_tiger where place_id = ".$iPlaceID.") as blub1) as blub2";
-        } elseif ($bIsInterpolation) {
-            $sSQL = "select place_id, partition, 'W' as osm_type, osm_id, 'place' as class, 'house' as type, null admin_level, housenumber, postcode,";
-            $sSQL .= " country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
-            $sSQL .= " (0.75-(30::float/40)) as importance, null as indexed_status, null as indexed_date, null as wikipedia, country_code, ";
-            $sSQL .= " get_address_by_language(place_id, housenumber, $sLanguagePrefArraySQL) as langaddress,";
-            $sSQL .= " null as placename,";
-            $sSQL .= " null as ref,";
-            if ($this->bExtraTags) $sSQL .= " null as extra,";
-            if ($this->bNameDetails) $sSQL .= " null as names,";
-            $sSQL .= " ST_X(point) as lon, ST_Y(point) as lat from (select *, ST_LineInterpolatePoint(linegeo, (housenumber-startnumber::float)/(endnumber-startnumber)::float) as point from ";
-            $sSQL .= " (select *, ";
-            $sSQL .= " CASE WHEN interpolationtype='odd' THEN floor((".$fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2+1";
-            $sSQL .= " WHEN interpolationtype='even' THEN ((".$fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2";
-            $sSQL .= " WHEN interpolationtype='all' THEN (".$fInterpolFraction."*(endnumber-startnumber)+startnumber)::int";
-            $sSQL .= " END as housenumber";
-            $sSQL .= " from location_property_osmline where place_id = ".$iPlaceID.") as blub1) as blub2";
-            // testcase: interpolationtype=odd, startnumber=1000, endnumber=1006, fInterpolFraction=1 => housenumber=1007 => error in st_lineinterpolatepoint
-            // but this will never happen, because if the searched point is that close to the endnumber, the endnumber house will be directly taken from placex (in ReverseGeocode.php line 220)
-            // and not interpolated
-        } else {
-            $sSQL = "select placex.place_id, partition, osm_type, osm_id, class,";
-            $sSQL .= " type, admin_level, housenumber, postcode, country_code,";
-            $sSQL .= " parent_place_id, linked_place_id, rank_address, rank_search, ";
-            $sSQL .= " coalesce(importance,0.75-(rank_search::float/40)) as importance, indexed_status, indexed_date, wikipedia, country_code, ";
-            $sSQL .= " get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress,";
-            $sSQL .= " get_name_by_language(name, $sLanguagePrefArraySQL) as placename,";
-            $sSQL .= " get_name_by_language(name, ARRAY['ref']) as ref,";
-            if ($this->bExtraTags) $sSQL .= " hstore_to_json(extratags) as extra,";
-            if ($this->bNameDetails) $sSQL .= " hstore_to_json(name) as names,";
-            $sSQL .= " (case when centroid is null then st_y(st_centroid(geometry)) else st_y(centroid) end) as lat,";
-            $sSQL .= " (case when centroid is null then st_x(st_centroid(geometry)) else st_x(centroid) end) as lon";
-            $sSQL .= " from placex where place_id = ".$iPlaceID;
         }
 
-        $aPlace = chksql($this->oDB->getRow($sSQL), "Could not lookup place");
+        // postcode table
+        $sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_POSTCODE);
+        if ($sPlaceIDs) {
+            $sSQL = 'SELECT';
+            $sSQL .= "  'P' as osm_type,";
+            $sSQL .= '  (SELECT osm_id from placex p WHERE p.place_id = lp.parent_place_id) as osm_id,';
+            $sSQL .= "  'place' as class, 'postcode' as type,";
+            $sSQL .= '  null as admin_level, rank_search, rank_address,';
+            $sSQL .= '  place_id, parent_place_id,';
+            $sSQL .= '  null as housenumber,';
+            $sSQL .= '  country_code,';
+            $sSQL .= $this->langAddressSql('-1');
+            $sSQL .= '  postcode as placename,';
+            $sSQL .= '  postcode as ref,';
+            if ($this->bExtraTags) $sSQL .= 'null AS extra,';
+            if ($this->bNameDetails) $sSQL .= 'null AS names,';
+            $sSQL .= '  ST_x(geometry) AS lon, ST_y(geometry) AS lat,';
+            $sSQL .= '  (0.75-(rank_search::float/40)) AS importance, ';
+            $sSQL .= $this->addressImportanceSql('geometry', 'lp.parent_place_id');
+            $sSQL .= '  null AS extra_place ';
+            $sSQL .= 'FROM location_postcode lp';
+            $sSQL .= " WHERE place_id in ($sPlaceIDs) ";
+            $sSQL .= "   AND lp.rank_address between $iMinRank and $iMaxRank";
 
-        if (!$aPlace['place_id']) return null;
+            $aSubSelects[] = $sSQL;
-
-        if ($this->bAddressDetails) {
-            // to get addressdetails for tiger data, the housenumber is needed
-            $iHousenumber = ($bIsTiger || $bIsInterpolation) ? $aPlace['housenumber'] : -1;
-            $aPlace['aAddress'] = $this->getAddressNames($aPlace['place_id'], $iHousenumber);
         }
 
-        if ($this->bExtraTags) {
+        // All other tables are rank 30 only.
-            if ($aPlace['extra']) {
+        if ($iMaxRank == 30) {
-                $aPlace['sExtraTags'] = json_decode($aPlace['extra']);
+            // TIGER table
-            } else {
+            if (CONST_Use_US_Tiger_Data) {
-                $aPlace['sExtraTags'] = (object) array();
+                $sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_TIGER);
+                if ($sPlaceIDs) {
+                    $sHousenumbers = Result::sqlHouseNumberTable($aResults, Result::TABLE_TIGER);
+                    // Tiger search only if a housenumber was searched and if it was found
+                    // (realized through a join)
+                    $sSQL = ' SELECT ';
+                    $sSQL .= "     'T' AS osm_type, ";
+                    $sSQL .= '     (SELECT osm_id from placex p WHERE p.place_id=blub.parent_place_id) as osm_id, ';
+                    $sSQL .= "     'place' AS class, ";
+                    $sSQL .= "     'house' AS type, ";
+                    $sSQL .= '     null AS admin_level, ';
+                    $sSQL .= '     30 AS rank_search, ';
+                    $sSQL .= '     30 AS rank_address, ';
+                    $sSQL .= '     place_id, ';
+                    $sSQL .= '     parent_place_id, ';
+                    $sSQL .= '     housenumber_for_place as housenumber,';
+                    $sSQL .= "     'us' AS country_code, ";
+                    $sSQL .= $this->langAddressSql('housenumber_for_place');
+                    $sSQL .= '     null AS placename, ';
+                    $sSQL .= '     null AS ref, ';
+                    if ($this->bExtraTags) $sSQL .= 'null AS extra,';
+                    if ($this->bNameDetails) $sSQL .= 'null AS names,';
+                    $sSQL .= '     st_x(centroid) AS lon, ';
+                    $sSQL .= '     st_y(centroid) AS lat,';
+                    $sSQL .= '     -1.15 AS importance, ';
+                    $sSQL .= $this->addressImportanceSql('centroid', 'blub.parent_place_id');
+                    $sSQL .= '     null AS extra_place ';
+                    $sSQL .= ' FROM (';
+                    $sSQL .= '     SELECT place_id, ';    // interpolate the Tiger housenumbers here
+                    $sSQL .= '         ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) AS centroid, ';
+                    $sSQL .= '         parent_place_id, ';
+                    $sSQL .= '         housenumber_for_place';
+                    $sSQL .= '     FROM (';
+                    $sSQL .= '            location_property_tiger ';
+                    $sSQL .= '            JOIN (values '.$sHousenumbers.') AS housenumbers(place_id, housenumber_for_place) USING(place_id)) ';
+                    $sSQL .= '     WHERE ';
+                    $sSQL .= '         housenumber_for_place >= startnumber';
+                    $sSQL .= '         AND housenumber_for_place <= endnumber';
+                    $sSQL .= ' ) AS blub'; //postgres wants an alias here
+
+                    $aSubSelects[] = $sSQL;
+                }
+            }
+
+            // osmline - interpolated housenumbers
+            $sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_OSMLINE);
+            if ($sPlaceIDs) {
+                $sHousenumbers = Result::sqlHouseNumberTable($aResults, Result::TABLE_OSMLINE);
+                // interpolation line search only if a housenumber was searched
+                // (realized through a join)
+                $sSQL = 'SELECT ';
+                $sSQL .= "  'W' AS osm_type, ";
+                $sSQL .= '  osm_id, ';
+                $sSQL .= "  'place' AS class, ";
+                $sSQL .= "  'house' AS type, ";
+                $sSQL .= '  15 AS admin_level, ';
+                $sSQL .= '  30 AS rank_search, ';
+                $sSQL .= '  30 AS rank_address, ';
+                $sSQL .= '  place_id, ';
+                $sSQL .= '  parent_place_id, ';
+                $sSQL .= '  housenumber_for_place as housenumber,';
+                $sSQL .= '  country_code, ';
+                $sSQL .= $this->langAddressSql('housenumber_for_place');
+                $sSQL .= '  null AS placename, ';
+                $sSQL .= '  null AS ref, ';
+                if ($this->bExtraTags) $sSQL .= 'null AS extra, ';
+                if ($this->bNameDetails) $sSQL .= 'null AS names, ';
+                $sSQL .= '  st_x(centroid) AS lon, ';
+                $sSQL .= '  st_y(centroid) AS lat, ';
+                // slightly smaller than the importance for normal houses
+                $sSQL .= '  -0.1 AS importance, ';
+                $sSQL .= $this->addressImportanceSql('centroid', 'blub.parent_place_id');
+                $sSQL .= '  null AS extra_place ';
+                $sSQL .= '  FROM (';
+                $sSQL .= '     SELECT ';
+                $sSQL .= '         osm_id, ';
+                $sSQL .= '         place_id, ';
+                $sSQL .= '         country_code, ';
+                $sSQL .= '         CASE ';             // interpolate the housenumbers here
+                $sSQL .= '           WHEN startnumber != endnumber ';
+                $sSQL .= '           THEN ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) ';
+                $sSQL .= '           ELSE ST_LineInterpolatePoint(linegeo, 0.5) ';
+                $sSQL .= '         END as centroid, ';
+                $sSQL .= '         parent_place_id, ';
+                $sSQL .= '         housenumber_for_place ';
+                $sSQL .= '     FROM (';
+                $sSQL .= '            location_property_osmline ';
+                $sSQL .= '            JOIN (values '.$sHousenumbers.') AS housenumbers(place_id, housenumber_for_place) USING(place_id)';
+                $sSQL .= '          ) ';
+                $sSQL .= '     WHERE housenumber_for_place >= 0 ';
+                $sSQL .= '  ) as blub'; //postgres wants an alias here
+
+                $aSubSelects[] = $sSQL;
+            }
+
+            if (CONST_Use_Aux_Location_data) {
+                $sPlaceIDs = Result::joinIdsByTable($aResults, Result::TABLE_AUX);
+                if ($sPlaceIDs) {
+                    $sHousenumbers = Result::sqlHouseNumberTable($aResults, Result::TABLE_AUX);
+                    $sSQL = '  SELECT ';
+                    $sSQL .= "     'L' AS osm_type, ";
+                    $sSQL .= '     place_id AS osm_id, ';
+                    $sSQL .= "     'place' AS class,";
+                    $sSQL .= "     'house' AS type, ";
+                    $sSQL .= '     null AS admin_level, ';
+                    $sSQL .= '     30 AS rank_search,';
+                    $sSQL .= '     30 AS rank_address, ';
+                    $sSQL .= '     place_id,';
+                    $sSQL .= '     parent_place_id, ';
+                    $sSQL .= '     housenumber,';
+                    $sSQL .= "     'us' AS country_code, ";
+                    $sSQL .= $this->langAddressSql('-1');
+                    $sSQL .= '     null AS placename, ';
+                    $sSQL .= '     null AS ref, ';
+                    if ($this->bExtraTags) $sSQL .= 'null AS extra, ';
+                    if ($this->bNameDetails) $sSQL .= 'null AS names, ';
+                    $sSQL .= '     ST_X(centroid) AS lon, ';
+                    $sSQL .= '     ST_Y(centroid) AS lat, ';
+                    $sSQL .= '     -1.10 AS importance, ';
+                    $sSQL .= $this->addressImportanceSql(
+                        'centroid',
+                        'location_property_aux.parent_place_id'
+                    );
+                    $sSQL .= '     null AS extra_place ';
+                    $sSQL .= '  FROM location_property_aux ';
+                    $sSQL .= "  WHERE place_id in ($sPlaceIDs) ";
+
+                    $aSubSelects[] = $sSQL;
+                }
             }
         }
 
-        if ($this->bNameDetails) {
+        if (CONST_Debug) var_dump($aSubSelects);
-            if ($aPlace['names']) {
+
-                $aPlace['sNameDetails'] = json_decode($aPlace['names']);
+        if (!sizeof($aSubSelects)) {
-            } else {
+            return array();
-                $aPlace['sNameDetails'] = (object) array();
-            }
         }
 
+        $aPlaces = chksql(
+            $this->oDB->getAll(join(' UNION ', $aSubSelects)),
+            'Could not lookup place'
+        );
+
         $aClassType = getClassTypes();
-        $sAddressType = '';
+        foreach ($aPlaces as &$aPlace) {
-        $sClassType = $aPlace['class'].':'.$aPlace['type'].':'.$aPlace['admin_level'];
+            if ($this->bAddressDetails) {
-        if (isset($aClassType[$sClassType]) && isset($aClassType[$sClassType]['simplelabel'])) {
+                // to get addressdetails for tiger data, the housenumber is needed
-            $sAddressType = $aClassType[$aClassType]['simplelabel'];
+                $aPlace['aAddress'] = $this->getAddressNames(
-        } else {
+                    $aPlace['place_id'],
-            $sClassType = $aPlace['class'].':'.$aPlace['type'];
+                    $aPlace['housenumber']
-            if (isset($aClassType[$sClassType]) && isset($aClassType[$sClassType]['simplelabel']))
+                );
-                $sAddressType = $aClassType[$sClassType]['simplelabel'];
+            }
-            else $sAddressType = $aPlace['class'];
+
+            if ($this->bExtraTags) {
+                if ($aPlace['extra']) {
+                    $aPlace['sExtraTags'] = json_decode($aPlace['extra']);
+                } else {
+                    $aPlace['sExtraTags'] = (object) array();
+                }
+            }
+
+            if ($this->bNameDetails) {
+                if ($aPlace['names']) {
+                    $aPlace['sNameDetails'] = json_decode($aPlace['names']);
+                } else {
+                    $aPlace['sNameDetails'] = (object) array();
+                }
+            }
+
+            $sAddressType = '';
+            $sClassType = $aPlace['class'].':'.$aPlace['type'].':'.$aPlace['admin_level'];
+            if (isset($aClassType[$sClassType]) && isset($aClassType[$sClassType]['simplelabel'])) {
+                $sAddressType = $aClassType[$aClassType]['simplelabel'];
+            } else {
+                $sClassType = $aPlace['class'].':'.$aPlace['type'];
+                if (isset($aClassType[$sClassType]) && isset($aClassType[$sClassType]['simplelabel']))
+                    $sAddressType = $aClassType[$sClassType]['simplelabel'];
+                else $sAddressType = $aPlace['class'];
+            }
+
+            $aPlace['addresstype'] = $sAddressType;
         }
 
-        $aPlace['addresstype'] = $sAddressType;
+        if (CONST_Debug) var_dump($aPlaces);
 
-        return $aPlace;
+        return $aPlaces;
     }
 
-    public function getAddressDetails($iPlaceID, $bAll = false, $housenumber = -1)
+    private function getAddressDetails($iPlaceID, $bAll, $sHousenumber)
     {
-        $sLanguagePrefArraySQL = "ARRAY[".join(',', array_map("getDBQuoted", $this->aLangPrefOrder))."]";
+        $sSQL = 'SELECT *,';
-
+        $sSQL .= '  get_name_by_language(name,'.$this->aLangPrefOrderSql.') as localname';
-        $sSQL = "select *,get_name_by_language(name,$sLanguagePrefArraySQL) as localname from get_addressdata(".$iPlaceID.",".$housenumber.")";
+        $sSQL .= ' FROM get_addressdata('.$iPlaceID.','.$sHousenumber.')';
-        if (!$bAll) $sSQL .= " WHERE isaddress OR type = 'country_code'";
+        if (!$bAll) {
-        $sSQL .= " order by rank_address desc,isaddress desc";
+            $sSQL .= " WHERE isaddress OR type = 'country_code'";
+        }
+        $sSQL .= ' ORDER BY rank_address desc,isaddress DESC';
 
         return chksql($this->oDB->getAll($sSQL));
     }
 
-    public function getAddressNames($iPlaceID, $housenumber = -1)
+    public function getAddressNames($iPlaceID, $sHousenumber = null)
     {
-        $aAddressLines = $this->getAddressDetails($iPlaceID, false, $housenumber);
+        $aAddressLines = $this->getAddressDetails(
+            $iPlaceID,
+            false,
+            $sHousenumber === null ? -1 : $sHousenumber
+        );
 
         $aAddress = array();
         $aFallback = array();
@@ -261,22 +534,22 @@ class PlaceLookup
 
         if (CONST_Search_AreaPolygons) {
             // Get the bounding box and outline polygon
-            $sSQL  = "select place_id,0 as numfeatures,st_area(geometry) as area,";
+            $sSQL  = 'select place_id,0 as numfeatures,st_area(geometry) as area,';
-            $sSQL .= "ST_Y(centroid) as centrelat,ST_X(centroid) as centrelon,";
+            $sSQL .= 'ST_Y(centroid) as centrelat,ST_X(centroid) as centrelon,';
-            $sSQL .= "ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,";
+            $sSQL .= 'ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,';
-            $sSQL .= "ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon";
+            $sSQL .= 'ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon';
-            if ($this->bIncludePolygonAsGeoJSON) $sSQL .= ",ST_AsGeoJSON(geometry) as asgeojson";
+            if ($this->bIncludePolygonAsGeoJSON) $sSQL .= ',ST_AsGeoJSON(geometry) as asgeojson';
-            if ($this->bIncludePolygonAsKML) $sSQL .= ",ST_AsKML(geometry) as askml";
+            if ($this->bIncludePolygonAsKML) $sSQL .= ',ST_AsKML(geometry) as askml';
-            if ($this->bIncludePolygonAsSVG) $sSQL .= ",ST_AsSVG(geometry) as assvg";
+            if ($this->bIncludePolygonAsSVG) $sSQL .= ',ST_AsSVG(geometry) as assvg';
-            if ($this->bIncludePolygonAsText || $this->bIncludePolygonAsPoints) $sSQL .= ",ST_AsText(geometry) as astext";
+            if ($this->bIncludePolygonAsText || $this->bIncludePolygonAsPoints) $sSQL .= ',ST_AsText(geometry) as astext';
-            $sFrom = " from placex where place_id = ".$iPlaceID;
+            $sFrom = ' from placex where place_id = '.$iPlaceID;
             if ($this->fPolygonSimplificationThreshold > 0) {
-                $sSQL .= " from (select place_id,centroid,ST_SimplifyPreserveTopology(geometry,".$this->fPolygonSimplificationThreshold.") as geometry".$sFrom.") as plx";
+                $sSQL .= ' from (select place_id,centroid,ST_SimplifyPreserveTopology(geometry,'.$this->fPolygonSimplificationThreshold.') as geometry'.$sFrom.') as plx';
             } else {
                 $sSQL .= $sFrom;
             }
 
-            $aPointPolygon = chksql($this->oDB->getRow($sSQL), "Could not get outline");
+            $aPointPolygon = chksql($this->oDB->getRow($sSQL), 'Could not get outline');
 
             if ($aPointPolygon['place_id']) {
                 if ($aPointPolygon['centrelon'] !== null && $aPointPolygon['centrelat'] !== null) {

lib/Result.php

@@ -0,0 +1,60 @@
+<?php
+
+namespace Nominatim;
+
+/**
+ * A single result of a search operation or a reverse lookup.
+ *
+ * This object only contains the id of the result. It does not yet
+ * have any details needed to format the output document.
+ */
+class Result
+{
+    const TABLE_PLACEX = 0;
+    const TABLE_POSTCODE = 1;
+    const TABLE_OSMLINE = 2;
+    const TABLE_AUX = 3;
+    const TABLE_TIGER = 4;
+
+    /// Database table that contains the result.
+    public $iTable;
+    /// Id of the result.
+    public $iId;
+    /// House number (only for interpolation results).
+    public $iHouseNumber = -1;
+    /// Number of exact matches in address (address searches only).
+    public $iExactMatches = 0;
+    /// Subranking within the results (the higher the worse).
+    public $iResultRank = 0;
+
+
+    public function __construct($sId, $iTable = Result::TABLE_PLACEX)
+    {
+        $this->iTable = $iTable;
+        $this->iId = (int) $sId;
+    }
+
+    public static function joinIdsByTable($aResults, $iTable)
+    {
+        return join(',', array_keys(array_filter(
+            $aResults,
+            function ($aValue) use ($iTable) {
+                return $aValue->iTable == $iTable;
+            }
+        )));
+    }
+    public static function sqlHouseNumberTable($aResults, $iTable)
+    {
+        $sHousenumbers = '';
+        $sSep = '';
+        foreach ($aResults as $oResult) {
+            if ($oResult->iTable == $iTable) {
+                $sHousenumbers .= $sSep.'('.$oResult->iId.',';
+                $sHousenumbers .= $oResult->iHouseNumber.')';
+                $sSep = ',';
+            }
+        }
+
+        return $sHousenumbers;
+    }
+}

lib/ReverseGeocode.php

@@ -2,6 +2,8 @@
 
 namespace Nominatim;
 
+require_once(CONST_BasePath.'/lib/Result.php');
+
 class ReverseGeocode
 {
     protected $oDB;
@@ -53,38 +55,39 @@ class ReverseGeocode
     protected function lookupInterpolation($sPointSQL, $fSearchDiam)
     {
         $sSQL = 'SELECT place_id, parent_place_id, 30 as rank_search,';
-        $sSQL .= ' ST_LineLocatePoint(linegeo,'.$sPointSQL.') as fraction';
+        $sSQL .= '  ST_LineLocatePoint(linegeo,'.$sPointSQL.') as fraction,';
-        $sSQL .= ' , ST_Distance(linegeo,'.$sPointSQL.') as distance';
+        $sSQL .= '  startnumber, endnumber, interpolationtype,';
+        $sSQL .= '  ST_Distance(linegeo,'.$sPointSQL.') as distance';
         $sSQL .= ' FROM location_property_osmline';
         $sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', linegeo, '.$fSearchDiam.')';
         $sSQL .= ' and indexed_status = 0 and startnumber is not NULL ';
-        $sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', linegeo) ASC limit 1';
+        $sSQL .= ' ORDER BY distance ASC limit 1';
 
         return chksql(
             $this->oDB->getRow($sSQL),
-            "Could not determine closest housenumber on an osm interpolation line."
+            'Could not determine closest housenumber on an osm interpolation line.'
         );
     }
 
-    /* lookup()
-     * returns { place_id =>, type => '(osm|tiger)' }
-     * fails if no place was found
-     */
-
-
     public function lookup($fLat, $fLon, $bDoInterpolation = true)
     {
-        $sPointSQL = 'ST_SetSRID(ST_Point('.$fLon.','.$fLat.'),4326)';
+        return $this->lookupPoint(
+            'ST_SetSRID(ST_Point('.$fLon.','.$fLat.'),4326)',
+            $bDoInterpolation
+        );
+    }
+
+    public function lookupPoint($sPointSQL, $bDoInterpolation = true)
+    {
         $iMaxRank = $this->iMaxRank;
 
         // Find the nearest point
         $fSearchDiam = 0.0004;
-        $iPlaceID = null;
+        $oResult = null;
+        $aPlace = null;
         $fMaxAreaDistance = 1;
-        $bIsInUnitedStates = false;
+        $bIsTigerStreet = false;
-        $bPlaceIsTiger = false;
+        while ($oResult === null && $fSearchDiam < $fMaxAreaDistance) {
-        $bPlaceIsLine = false;
-        while (!$iPlaceID && $fSearchDiam < $fMaxAreaDistance) {
             $fSearchDiam = $fSearchDiam * 2;
 
             // If we have to expand the search area by a large amount then we need a larger feature
@@ -99,16 +102,14 @@ class ReverseGeocode
             if ($fSearchDiam > 0.001 && $iMaxRank > 26) {
                 // try with interpolations before continuing
                 if ($bDoInterpolation) {
-                    // no house found, try with interpolations
+                    $aHouse = $this->lookupInterpolation($sPointSQL, $fSearchDiam/2);
-                    $aPlaceLine = $this->lookupInterpolation($sPointSQL, $fSearchDiam/2);
 
-                    if ($aPlaceLine) {
+                    if ($aHouse) {
-                        // interpolation is closer to point than placex house
+                        $oResult = new Result($aHouse['place_id'], Result::TABLE_OSMLINE);
-                        $bPlaceIsLine = true;
+                        $oResult->iHouseNumber = closestHouseNumber($aHouse);
-                        $aPlace = $aPlaceLine;
+
-                        $iPlaceID = $aPlaceLine['place_id'];
+                        $aPlace = $aHouse;
-                        $iParentPlaceID = $aPlaceLine['parent_place_id']; // the street
+                        $iParentPlaceID = $aHouse['parent_place_id']; // the street
-                        $fFraction = $aPlaceLine['fraction'];
                         $iMaxRank = 30;
 
                         break;
@@ -118,106 +119,105 @@ class ReverseGeocode
                 $iMaxRank = 26;
             }
 
-            $sSQL = 'select place_id,parent_place_id,rank_search,country_code';
+            $sSQL = 'select place_id,parent_place_id,rank_search,country_code,';
-            $sSQL .= ' FROM placex';
+            $sSQL .= '  ST_distance('.$sPointSQL.', geometry) as distance';
-            $sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')';
+            $sSQL .= ' FROM ';
-            $sSQL .= ' and rank_search != 28 and rank_search >= '.$iMaxRank;
+            if ($fSearchDiam < 0.01) {
-            $sSQL .= ' and (name is not null or housenumber is not null)';
+                $sSQL .= ' placex';
+                $sSQL .= '   WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')';
+                $sSQL .= '   AND';
+            } else {
+                $sSQL .= ' (SELECT * FROM placex ';
+                $sSQL .= '   WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')';
+                $sSQL .= '   LIMIT 1000) as p WHERE';
+            }
+            $sSQL .= ' rank_search != 28 and rank_search >= '.$iMaxRank;
+            $sSQL .= ' and (name is not null or housenumber is not null';
+            $sSQL .= '      or rank_search between 26 and 27)';
             $sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\',\'man_made\')';
-            $sSQL .= ' and indexed_status = 0 ';
+            $sSQL .= ' and indexed_status = 0 and linked_place_id is null';
             $sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') ';
             $sSQL .= ' OR ST_DWithin('.$sPointSQL.', centroid, '.$fSearchDiam.'))';
-            $sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1';
+            $sSQL .= ' ORDER BY distance ASC limit 1';
             if (CONST_Debug) var_dump($sSQL);
             $aPlace = chksql(
                 $this->oDB->getRow($sSQL),
-                "Could not determine closest place."
+                'Could not determine closest place.'
             );
-            $iPlaceID = $aPlace['place_id'];
+            if ($aPlace) {
-            $iParentPlaceID = $aPlace['parent_place_id'];
+                $oResult = new Result($aPlace['place_id']);
-            $bIsInUnitedStates = ($aPlace['country_code'] == 'us');
+                $iParentPlaceID = $aPlace['parent_place_id'];
-        }
+                if ($bDoInterpolation) {
+                    if ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27) {
+                        $bIsTigerStreet = ($aPlace['country_code'] == 'us');
+                    } elseif ($aPlace['rank_search'] == 30) {
+                        // If a house was found, make sure there isn't an
+                        // interpolation line that is closer.
+                        $aHouse = $this->lookupInterpolation(
+                            $sPointSQL,
+                            $aPlace['distance']
+                        );
+                        if ($aHouse && $aPlace['distance'] < $aHouse['distance']) {
+                            $oResult = new Result(
+                                $aHouse['place_id'],
+                                Result::TABLE_OSMLINE
+                            );
+                            $oResult->iHouseNumber = closestHouseNumber($aHouse);
 
-        // If a house was found make sure there isn't an interpolation line
+                            $aPlace = $aHouse;
-        // that is closer
+                            $iParentPlaceID = $aHouse['parent_place_id'];
-        if ($bDoInterpolation && !$bPlaceIsLine && $aPlace && $aPlace['rank_search'] == 30) {
+                        }
-            // get the distance of the house to the search point
+                    }
-            $sSQL = 'SELECT ST_distance('.$sPointSQL.', house.geometry)';
-            $sSQL .= ' FROM placex as house WHERE house.place_id='.$iPlaceID;
-
-            $fDistancePlacex = chksql(
-                $this->oDB->getOne($sSQL),
-                "Could not determine distance between searched point and placex house."
-            );
-
-            // look for an interpolation that is closer
-            $aPlaceLine = $this->lookupInterpolation($sPointSQL, $fDistancePlacex);
-
-            if ($aPlaceLine && (float) $aPlaceLine['distance'] < (float) $fDistancePlacex) {
-                // interpolation is closer to point than placex house
-                $bPlaceIsLine = true;
-                $aPlace = $aPlaceLine;
-                $iPlaceID = $aPlaceLine['place_id'];
-                $iParentPlaceID = $aPlaceLine['parent_place_id']; // the street
-                $fFraction = $aPlaceLine['fraction'];
-            }
-        }
-
-        // Only street found? If it's in the US we can check TIGER data for nearest housenumber
-        if (CONST_Use_US_Tiger_Data && $bDoInterpolation && $bIsInUnitedStates && $this->iMaxRank >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 )) {
-            $fSearchDiam = 0.001;
-            $sSQL = 'SELECT place_id,parent_place_id,30 as rank_search, ST_LineLocatePoint(linegeo,'.$sPointSQL.') as fraction';
-            //if (CONST_Debug) { $sSQL .= ', housenumber, ST_distance('.$sPointSQL.', centroid) as distance, st_y(centroid) as lat, st_x(centroid) as lon'; }
-            $sSQL .= ' FROM location_property_tiger WHERE parent_place_id = '.$iPlaceID;
-            $sSQL .= ' AND ST_DWithin('.$sPointSQL.', linegeo, '.$fSearchDiam.')';  //no centroid anymore in Tiger data, now we have lines
-            $sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', linegeo) ASC limit 1';
-
-            if (CONST_Debug) {
-                $sSQL = preg_replace('/limit 1/', 'limit 100', $sSQL);
-                var_dump($sSQL);
-
-                $aAllHouses = chksql($this->oDB->getAll($sSQL));
-                foreach ($aAllHouses as $i) {
-                    echo $i['housenumber'] . ' | ' . $i['distance'] * 1000 . ' | ' . $i['lat'] . ' | ' . $i['lon']. ' | '. "<br>\n";
                 }
             }
+        }
+
+        // Only street found? In the US we can check TIGER data for nearest housenumber
+        if (CONST_Use_US_Tiger_Data && $bIsTigerStreet && $this->iMaxRank >= 28) {
+            $fSearchDiam = $aPlace['rank_search'] > 28 ? $aPlace['distance'] : 0.001;
+            $sSQL = 'SELECT place_id,parent_place_id,30 as rank_search,';
+            $sSQL .= 'ST_LineLocatePoint(linegeo,'.$sPointSQL.') as fraction,';
+            $sSQL .= 'ST_distance('.$sPointSQL.', linegeo) as distance,';
+            $sSQL .= 'startnumber,endnumber,interpolationtype';
+            $sSQL .= ' FROM location_property_tiger WHERE parent_place_id = '.$oResult->iId;
+            $sSQL .= ' AND ST_DWithin('.$sPointSQL.', linegeo, '.$fSearchDiam.')';
+            $sSQL .= ' ORDER BY distance ASC limit 1';
+
+            if (CONST_Debug) var_dump($sSQL);
 
             $aPlaceTiger = chksql(
                 $this->oDB->getRow($sSQL),
-                "Could not determine closest Tiger place."
+                'Could not determine closest Tiger place.'
             );
             if ($aPlaceTiger) {
                 if (CONST_Debug) var_dump('found Tiger housenumber', $aPlaceTiger);
-                $bPlaceIsTiger = true;
                 $aPlace = $aPlaceTiger;
-                $iPlaceID = $aPlaceTiger['place_id'];
+                $oResult = new Result($aPlace['place_id'], Result::TABLE_TIGER);
-                $iParentPlaceID = $aPlaceTiger['parent_place_id']; // the street
+                $oResult->iHouseNumber = closestHouseNumber($aPlaceTiger);
-                $fFraction = $aPlaceTiger['fraction'];
+                $iParentPlaceID = $aPlace['parent_place_id'];
                 $iMaxRank = 30;
             }
         }
 
         // The point we found might be too small - use the address to find what it is a child of
-        if ($iPlaceID && $iMaxRank < 28) {
+        if ($oResult !== null && $iMaxRank < 28) {
-            if (($aPlace['rank_search'] > 28 || $bPlaceIsTiger || $bPlaceIsLine) && $iParentPlaceID) {
+            if ($aPlace['rank_search'] > 28 && $iParentPlaceID) {
                 $iPlaceID = $iParentPlaceID;
-                $bPlaceIsLine = false;
+            } else {
-                $bPlaceIsTiger = false;
+                $iPlaceID = $oResult->iId;
             }
-            $sSQL  = 'select address_place_id';
+            $sSQL  = 'select a.address_place_id';
-            $sSQL .= ' FROM place_addressline';
+            $sSQL .= ' FROM place_addressline a, placex p';
-            $sSQL .= " WHERE place_id = $iPlaceID";
+            $sSQL .= " WHERE a.place_id = $iPlaceID and a.address_place_id = p.place_id";
+            $sSQL .= '   AND p.linked_place_id is null';
             $sSQL .= " ORDER BY abs(cached_rank_address - $iMaxRank) asc,cached_rank_address desc,isaddress desc,distance desc";
             $sSQL .= ' LIMIT 1';
-            $iPlaceID = chksql($this->oDB->getOne($sSQL), "Could not get parent for place.");
+            $iPlaceID = chksql($this->oDB->getOne($sSQL), 'Could not get parent for place.');
-            if (!$iPlaceID) {
+            if ($iPlaceID) {
-                $iPlaceID = $aPlace['place_id'];
+                $oResult = new Result($iPlaceID);
             }
         }
-        return array(
+
-                'place_id' => $iPlaceID,
+        return $oResult;
-                'type' => $bPlaceIsTiger ? 'tiger' : ($bPlaceIsLine ? 'interpolation' : 'osm'),
-                'fraction' => ($bPlaceIsTiger || $bPlaceIsLine) ? $fFraction : -1
-               );
     }
 }

lib/SearchContext.php

@@ -0,0 +1,270 @@
+<?php
+
+namespace Nominatim;
+
+require_once(CONST_BasePath.'/lib/lib.php');
+
+
+/**
+ * Collection of search constraints that are independent of the
+ * actual interpretation of the search query.
+ *
+ * The search context is shared between all SearchDescriptions. This
+ * object mainly serves as context provider for the database queries.
+ * Therefore most data is directly cached as SQL statements.
+ */
+class SearchContext
+{
+    /// Search radius around a given Near reference point.
+    private $fNearRadius = false;
+    /// True if search must be restricted to viewbox only.
+    public $bViewboxBounded = false;
+
+    /// Reference point for search (as SQL).
+    public $sqlNear = '';
+    /// Viewbox selected for search (as SQL).
+    public $sqlViewboxSmall = '';
+    /// Viewbox with a larger buffer around (as SQL).
+    public $sqlViewboxLarge = '';
+    /// Reference along a route (as SQL).
+    public $sqlViewboxCentre = '';
+    /// List of countries to restrict search to (as SQL).
+    public $sqlCountryList = '';
+    /// List of place IDs to exclude (as SQL).
+    private $sqlExcludeList = '';
+
+
+    /**
+     * Check if a reference point is defined.
+     *
+     * @return bool True if a reference point is defined.
+     */
+    public function hasNearPoint()
+    {
+        return $this->fNearRadius !== false;
+    }
+
+    /**
+     * Get radius around reference point.
+     *
+     * @return float Search radius around refernce point.
+     */
+    public function nearRadius()
+    {
+        return $this->fNearRadius;
+    }
+
+    /**
+     * Set search reference point in WGS84.
+     *
+     * If set, then only places around this point will be taken into account.
+     *
+     * @param float $fLat    Latitude of point.
+     * @param float $fLon    Longitude of point.
+     * @param float $fRadius Search radius around point.
+     *
+     * @return void
+     */
+    public function setNearPoint($fLat, $fLon, $fRadius = 0.1)
+    {
+        $this->fNearRadius = $fRadius;
+        $this->sqlNear = 'ST_SetSRID(ST_Point('.$fLon.','.$fLat.'),4326)';
+    }
+
+    /**
+     * Check if the search is geographically restricted.
+     *
+     * Searches are restricted if a reference point is given or if
+     * a bounded viewbox is set.
+     *
+     * @return bool True, if the search is geographically bounded.
+     */
+    public function isBoundedSearch()
+    {
+        return $this->hasNearPoint() || ($this->sqlViewboxSmall && $this->bViewboxBounded);
+    }
+
+    /**
+     * Set rectangular viewbox.
+     *
+     * The viewbox may be bounded which means that no search results
+     * must be outside the viewbox.
+     *
+     * @param float[4] $aViewBox Coordinates of the viewbox.
+     * @param bool     $bBounded True if the viewbox is bounded.
+     *
+     * @return void
+     */
+    public function setViewboxFromBox(&$aViewBox, $bBounded)
+    {
+        $this->bViewboxBounded = $bBounded;
+        $this->sqlViewboxCentre = '';
+
+        $this->sqlViewboxSmall = sprintf(
+            'ST_SetSRID(ST_MakeBox2D(ST_Point(%F,%F),ST_Point(%F,%F)),4326)',
+            $aViewBox[0],
+            $aViewBox[1],
+            $aViewBox[2],
+            $aViewBox[3]
+        );
+
+        $fHeight = $aViewBox[0] - $aViewBox[2];
+        $fWidth = $aViewBox[1] - $aViewBox[3];
+
+        $this->sqlViewboxLarge = sprintf(
+            'ST_SetSRID(ST_MakeBox2D(ST_Point(%F,%F),ST_Point(%F,%F)),4326)',
+            max($aViewBox[0], $aViewBox[2]) + $fHeight,
+            max($aViewBox[1], $aViewBox[3]) + $fWidth,
+            min($aViewBox[0], $aViewBox[2]) - $fHeight,
+            min($aViewBox[1], $aViewBox[3]) - $fWidth
+        );
+    }
+
+    /**
+     * Set viewbox along a route.
+     *
+     * The viewbox may be bounded which means that no search results
+     * must be outside the viewbox.
+     *
+     * @param object   $oDB          DB connection to use for computing the box.
+     * @param string[] $aRoutePoints List of x,y coordinates along a route.
+     * @param float    $fRouteWidth  Buffer around the route to use.
+     * @param bool     $bBounded     True if the viewbox bounded.
+     *
+     * @return void
+     */
+    public function setViewboxFromRoute(&$oDB, $aRoutePoints, $fRouteWidth, $bBounded)
+    {
+        $this->bViewboxBounded = $bBounded;
+        $this->sqlViewboxCentre = "ST_SetSRID('LINESTRING(";
+        $sSep = '';
+        foreach ($aRoutePoints as $aPoint) {
+            $fPoint = (float)$aPoint;
+            $this->sqlViewboxCentre .= $sSep.$fPoint;
+            $sSep = ($sSep == ' ') ? ',' : ' ';
+        }
+        $this->sqlViewboxCentre .= ")'::geometry,4326)";
+
+        $sSQL = 'ST_BUFFER('.$this->sqlViewboxCentre.','.($fRouteWidth/69).')';
+        $sGeom = chksql($oDB->getOne('select '.$sSQL), 'Could not get small viewbox');
+        $this->sqlViewboxSmall = "'".$sGeom."'::geometry";
+
+        $sSQL = 'ST_BUFFER('.$this->sqlViewboxCentre.','.($fRouteWidth/30).')';
+        $sGeom = chksql($oDB->getOne('select '.$sSQL), 'Could not get large viewbox');
+        $this->sqlViewboxLarge = "'".$sGeom."'::geometry";
+    }
+
+    /**
+     * Set list of excluded place IDs.
+     *
+     * @param integer[] $aExcluded List of IDs.
+     *
+     * @return void
+     */
+    public function setExcludeList($aExcluded)
+    {
+        $this->sqlExcludeList = ' not in ('.join(',', $aExcluded).')';
+    }
+
+    /**
+     * Set list of countries to restrict search to.
+     *
+     * @param string[] $aCountries List of two-letter lower-case country codes.
+     *
+     * @return void
+     */
+    public function setCountryList($aCountries)
+    {
+        $this->sqlCountryList = '('.join(',', array_map('addQuotes', $aCountries)).')';
+    }
+
+    /**
+     * Extract a reference point from a query string.
+     *
+     * @param string $sQuery Query to scan.
+     *
+     * @return string The remaining query string.
+     */
+    public function setNearPointFromQuery($sQuery)
+    {
+        $aResult = parseLatLon($sQuery);
+
+        if ($aResult !== false
+            && $aResult[1] <= 90.1
+            && $aResult[1] >= -90.1
+            && $aResult[2] <= 180.1
+            && $aResult[2] >= -180.1
+        ) {
+            $this->setNearPoint($aResult[1], $aResult[2]);
+            $sQuery = trim(str_replace($aResult[0], ' ', $sQuery));
+        }
+
+        return $sQuery;
+    }
+
+    /**
+     * Get an SQL snipped for computing the distance from the reference point.
+     *
+     * @param string $sObj SQL variable name to compute the distance from.
+     *
+     * @return string An SQL string.
+     */
+    public function distanceSQL($sObj)
+    {
+        return 'ST_Distance('.$this->sqlNear.", $sObj)";
+    }
+
+    /**
+     * Get an SQL snipped for checking if something is within range of the
+     * reference point.
+     *
+     * @param string $sObj SQL variable name to compute if it is within range.
+     *
+     * @return string An SQL string.
+     */
+    public function withinSQL($sObj)
+    {
+        return sprintf('ST_DWithin(%s, %s, %F)', $sObj, $this->sqlNear, $this->fNearRadius);
+    }
+
+    /**
+     * Get an SQL snipped of the importance factor of the viewbox.
+     *
+     * The importance factor is computed by checking if an object is within
+     * the viewbox and/or the extended version of the viewbox.
+     *
+     * @param string $sObj SQL variable name of object to weight the importance
+     *
+     * @return string SQL snipped of the factor with a leading multiply sign.
+     */
+    public function viewboxImportanceSQL($sObj)
+    {
+        $sSQL = '';
+
+        if ($this->sqlViewboxSmall) {
+            $sSQL = " * CASE WHEN ST_Contains($this->sqlViewboxSmall, $sObj) THEN 1 ELSE 0.5 END";
+        }
+        if ($this->sqlViewboxLarge) {
+            $sSQL = " * CASE WHEN ST_Contains($this->sqlViewboxLarge, $sObj) THEN 1 ELSE 0.5 END";
+        }
+
+        return $sSQL;
+    }
+
+    /**
+     * SQL snipped checking if a place ID should be excluded.
+     *
+     * @param string $sVariable SQL variable name of place ID to check,
+     *                          potentially prefixed with more SQL.
+     *
+     * @return string SQL snippet.
+     */
+    public function excludeSQL($sVariable)
+    {
+        if ($this->sqlExcludeList) {
+            return $sVariable.$this->sqlExcludeList;
+        }
+
+        return '';
+    }
+}

lib/SpecialSearchOperator.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace Nominatim;
+
+/**
+ * Operators describing special searches.
+ */
+abstract class Operator
+{
+    /// No operator selected.
+    const NONE = 0;
+    /// Search for POI of the given type.
+    const TYPE = 1;
+    /// Search for POIs near the given place.
+    const NEAR = 2;
+    /// Search for POIS in the given place.
+    const IN = 3;
+    /// Search for POIS named as given.
+    const NAME = 4;
+    /// Search for postcodes.
+    const POSTCODE = 5;
+
+    private static $aConstantNames = null;
+
+
+    public static function toString($iOperator)
+    {
+        if ($iOperator == Operator::NONE) {
+            return '';
+        }
+
+        if (Operator::$aConstantNames === null) {
+            $oReflector = new \ReflectionClass('Nominatim\Operator');
+            $aConstants = $oReflector->getConstants();
+
+            Operator::$aConstantNames = array();
+            foreach ($aConstants as $sName => $iValue) {
+                Operator::$aConstantNames[$iValue] = $sName;
+            }
+        }
+
+        return Operator::$aConstantNames[$iOperator];
+    }
+}

lib/cmd.php

@@ -99,7 +99,7 @@ function showUsage($aSpec, $bExit = false, $sError = false)
         echo 'Try `'.basename($_SERVER['argv'][0]).' --help` for more information.'."\n";
         exit;
     }
-    echo "Usage: ".basename($_SERVER['argv'][0])."\n";
+    echo 'Usage: '.basename($_SERVER['argv'][0])."\n";
     $bFirst = true;
     foreach ($aSpec as $aLine) {
         if (is_array($aLine)) {
@@ -128,3 +128,61 @@ function chksql($oSql, $sMsg = false)
 
     return $oSql;
 }
+
+function info($sMsg)
+{
+    echo date('Y-m-d H:i:s == ').$sMsg."\n";
+}
+
+$aWarnings = [];
+
+
+function warn($sMsg)
+{
+    $GLOBALS['aWarnings'][] = $sMsg;
+    echo date('Y-m-d H:i:s == ').'WARNING: '.$sMsg."\n";
+}
+
+
+function repeatWarnings()
+{
+    foreach ($GLOBALS['aWarnings'] as $sMsg) {
+        echo '  * ',$sMsg."\n";
+    }
+}
+
+
+function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreErrors = false)
+{
+    // Convert database DSN to psql parameters
+    $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
+    if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
+    $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
+    if (!$bVerbose) {
+        $sCMD .= ' -q';
+    }
+    if ($bfatal && !$bIgnoreErrors) {
+        $sCMD .= ' -v ON_ERROR_STOP=1';
+    }
+    $aDescriptors = array(
+                     0 => array('pipe', 'r'),
+                     1 => STDOUT,
+                     2 => STDERR
+                    );
+    $ahPipes = null;
+    $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
+    if (!is_resource($hProcess)) {
+        fail('unable to start pgsql');
+    }
+
+    while (strlen($sScript)) {
+        $written = fwrite($ahPipes[0], $sScript);
+        if ($written <= 0) break;
+        $sScript = substr($sScript, $written);
+    }
+    fclose($ahPipes[0]);
+    $iReturn = proc_close($hProcess);
+    if ($bfatal && $iReturn > 0) {
+        fail("pgsql returned with error code ($iReturn)");
+    }
+}

lib/db.php

@@ -8,7 +8,7 @@ function &getDB($bNew = false, $bPersistent = false)
     // Get the database object
     $oDB = chksql(
         DB::connect(CONST_Database_DSN.($bNew?'?new_link=true':''), $bPersistent),
-        "Failed to establish database connection"
+        'Failed to establish database connection'
     );
     $oDB->setFetchMode(DB_FETCHMODE_ASSOC);
     $oDB->query("SET DateStyle TO 'sql,european'");
@@ -23,6 +23,11 @@ function getDBQuoted($s)
     return "'".pg_escape_string($s)."'";
 }
 
+function getArraySQL($a)
+{
+    return 'ARRAY['.join(',', $a).']';
+}
+
 function getPostgresVersion(&$oDB)
 {
     $sVersionString = $oDB->getOne('select version()');

lib/init-website.php

@@ -10,7 +10,7 @@ require_once('ParameterParser.php');
  */
 
 
-function chksql($oSql, $sMsg = "Database request failed")
+function chksql($oSql, $sMsg = 'Database request failed')
 {
     if (!PEAR::isError($oSql)) return $oSql;
 
@@ -38,10 +38,10 @@ INTERNALFAIL;
     if (CONST_Debug) {
         var_dump($oSql);
     } else {
-        echo "<pre>\n".$oSql->getUserInfo()."</pre>";
+        echo "<pre>\n".$oSql->getUserInfo().'</pre>';
     }
 
-    echo "</pre></p></body></html>";
+    echo '</pre></p></body></html>';
     exit;
 }
 
@@ -49,20 +49,20 @@ function failInternalError($sError, $sSQL = false, $vDumpVar = false)
 {
     header('HTTP/1.0 500 Internal Server Error');
     header('Content-type: text/html; charset=utf-8');
-    echo "<html><body><h1>Internal Server Error</h1>";
+    echo '<html><body><h1>Internal Server Error</h1>';
     echo '<p>Nominatim has encountered an internal error while processing your request. This is most likely because of a bug in the software.</p>';
-    echo "<p><b>Details:</b> ".$sError,"</p>";
+    echo '<p><b>Details:</b> '.$sError,'</p>';
     echo '<p>Feel free to file an issue on <a href="https://github.com/openstreetmap/Nominatim/issues">Github</a>. ';
     echo 'Please include the error message above and the URL you used.</p>';
     if (CONST_Debug) {
-        echo "<hr><h2>Debugging Information</h2><br>";
+        echo '<hr><h2>Debugging Information</h2><br>';
         if ($sSQL) {
-            echo "<h3>SQL query</h3><code>".$sSQL."</code>";
+            echo '<h3>SQL query</h3><code>'.$sSQL.'</code>';
         }
         if ($vDumpVar) {
-            echo "<h3>Result</h3> <code>";
+            echo '<h3>Result</h3> <code>';
             var_dump($vDumpVar);
-            echo "</code>";
+            echo '</code>';
         }
     }
     echo "\n</body></html>\n";
@@ -74,9 +74,9 @@ function userError($sError)
 {
     header('HTTP/1.0 400 Bad Request');
     header('Content-type: text/html; charset=utf-8');
-    echo "<html><body><h1>Bad Request</h1>";
+    echo '<html><body><h1>Bad Request</h1>';
     echo '<p>Nominatim has encountered an error with your request.</p>';
-    echo "<p><b>Details:</b> ".$sError."</p>";
+    echo '<p><b>Details:</b> '.$sError.'</p>';
     echo '<p>If you feel this error is incorrect feel file an issue on <a href="https://github.com/openstreetmap/Nominatim/issues">Github</a>. ';
     echo 'Please include the error message above and the URL you used.</p>';
     echo "\n</body></html>\n";
@@ -89,10 +89,10 @@ function userError($sError)
  */
 
 if (CONST_NoAccessControl) {
-    header("Access-Control-Allow-Origin: *");
+    header('Access-Control-Allow-Origin: *');
-    header("Access-Control-Allow-Methods: OPTIONS,GET");
+    header('Access-Control-Allow-Methods: OPTIONS,GET');
     if (!empty($_SERVER['HTTP_ACCESS_CONTROL_REQUEST_HEADERS'])) {
-        header("Access-Control-Allow-Headers: ".$_SERVER['HTTP_ACCESS_CONTROL_REQUEST_HEADERS']);
+        header('Access-Control-Allow-Headers: '.$_SERVER['HTTP_ACCESS_CONTROL_REQUEST_HEADERS']);
     }
 }
 if ($_SERVER['REQUEST_METHOD'] == 'OPTIONS') exit;

lib/lib.php

@@ -38,7 +38,7 @@ function getDatabaseDate(&$oDB)
     // Find the newest node in the DB
     $iLastOSMID = $oDB->getOne("select max(osm_id) from place where osm_type = 'N'");
     // Lookup the timestamp that node was created
-    $sLastNodeURL = 'https://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
+    $sLastNodeURL = 'https://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID.'/1';
     $sLastNodeXML = file_get_contents($sLastNodeURL);
 
     if ($sLastNodeXML === false) {
@@ -51,14 +51,6 @@ function getDatabaseDate(&$oDB)
 }
 
 
-function bySearchRank($a, $b)
-{
-    if ($a['iSearchRank'] == $b['iSearchRank'])
-        return strlen($a['sOperator']) + strlen($a['sHouseNumber']) - strlen($b['sOperator']) - strlen($b['sHouseNumber']);
-    return ($a['iSearchRank'] < $b['iSearchRank']?-1:1);
-}
-
-
 function byImportance($a, $b)
 {
     if ($a['importance'] != $b['importance'])
@@ -68,73 +60,6 @@ function byImportance($a, $b)
 }
 
 
-function getWordSets($aWords, $iDepth)
-{
-    $aResult = array(array(join(' ', $aWords)));
-    $sFirstToken = '';
-    if ($iDepth < 7) {
-        while (sizeof($aWords) > 1) {
-            $sWord = array_shift($aWords);
-            $sFirstToken .= ($sFirstToken?' ':'').$sWord;
-            $aRest = getWordSets($aWords, $iDepth+1);
-            foreach ($aRest as $aSet) {
-                $aResult[] = array_merge(array($sFirstToken), $aSet);
-            }
-        }
-    }
-    return $aResult;
-}
-
-function getInverseWordSets($aWords, $iDepth)
-{
-    $aResult = array(array(join(' ', $aWords)));
-    $sFirstToken = '';
-    if ($iDepth < 8) {
-        while (sizeof($aWords) > 1) {
-            $sWord = array_pop($aWords);
-            $sFirstToken = $sWord.($sFirstToken?' ':'').$sFirstToken;
-            $aRest = getInverseWordSets($aWords, $iDepth+1);
-            foreach ($aRest as $aSet) {
-                $aResult[] = array_merge(array($sFirstToken), $aSet);
-            }
-        }
-    }
-    return $aResult;
-}
-
-
-function getTokensFromSets($aSets)
-{
-    $aTokens = array();
-    foreach ($aSets as $aSet) {
-        foreach ($aSet as $sWord) {
-            $aTokens[' '.$sWord] = ' '.$sWord;
-            $aTokens[$sWord] = $sWord;
-        }
-    }
-    return $aTokens;
-}
-
-
-function gbPostcodeCalculate($sPostcode, $sPostcodeSector, $sPostcodeEnd, &$oDB)
-{
-    // Try an exact match on the gb_postcode table
-    $sSQL = 'select \'AA\', ST_X(ST_Centroid(geometry)) as lon,ST_Y(ST_Centroid(geometry)) as lat from gb_postcode where postcode = \''.$sPostcode.'\'';
-    $aNearPostcodes = chksql($oDB->getAll($sSQL));
-
-    if (sizeof($aNearPostcodes)) {
-        $aPostcodes = array();
-        foreach ($aNearPostcodes as $aPostcode) {
-            $aPostcodes[] = array('lat' => $aPostcode['lat'], 'lon' => $aPostcode['lon'], 'radius' => 0.005);
-        }
-
-        return $aPostcodes;
-    }
-
-    return false;
-}
-
-
 function getClassTypes()
 {
     return array(
@@ -488,11 +413,11 @@ function javascript_renderData($xVal, $iOptions = 0)
     $jsonout = json_encode($xVal, $iOptions);
 
     if (!isset($_GET['json_callback'])) {
-        header("Content-Type: application/json; charset=UTF-8");
+        header('Content-Type: application/json; charset=UTF-8');
         echo $jsonout;
     } else {
         if (preg_match('/^[$_\p{L}][$_\p{L}\p{Nd}.[\]]*$/u', $_GET['json_callback'])) {
-            header("Content-Type: application/javascript; charset=UTF-8");
+            header('Content-Type: application/javascript; charset=UTF-8');
             echo $_GET['json_callback'].'('.$jsonout.')';
         } else {
             header('HTTP/1.0 400 Bad Request');
@@ -508,69 +433,22 @@ function _debugDumpGroupedSearches($aData, $aTokens)
         foreach ($aTokens as $sToken => $aWords) {
             if ($aWords) {
                 foreach ($aWords as $aToken) {
-                    $aWordsIDs[$aToken['word_id']] = $sToken.'('.$aToken['word_id'].')';
+                    $aWordsIDs[$aToken['word_id']] =
+                        '#'.$sToken.'('.$aToken['word_id'].')#';
                 }
             }
         }
     }
-    echo "<table border=\"1\">";
+    echo '<table border="1">';
-    echo "<tr><th>rank</th><th>Name Tokens</th><th>Name Not</th>";
+    echo '<tr><th>rank</th><th>Name Tokens</th><th>Name Not</th>';
-    echo "<th>Address Tokens</th><th>Address Not</th><th>country</th>";
+    echo '<th>Address Tokens</th><th>Address Not</th><th>country</th><th>operator</th>';
-    echo "<th>operator</th><th>class</th><th>type</th><th>house#</th>";
+    echo '<th>class</th><th>type</th><th>postcode</th><th>housenumber</th></tr>';
-    echo "<th>Lat</th><th>Lon</th><th>Radius</th></tr>";
     foreach ($aData as $iRank => $aRankedSet) {
         foreach ($aRankedSet as $aRow) {
-            echo "<tr>";
+            $aRow->dumpAsHtmlTableRow($aWordsIDs);
-            echo "<td>$iRank</td>";
-
-            echo "<td>";
-            $sSep = '';
-            foreach ($aRow['aName'] as $iWordID) {
-                echo $sSep.'#'.$aWordsIDs[$iWordID].'#';
-                $sSep = ', ';
-            }
-            echo "</td>";
-
-            echo "<td>";
-            $sSep = '';
-            foreach ($aRow['aNameNonSearch'] as $iWordID) {
-                echo $sSep.'#'.$aWordsIDs[$iWordID].'#';
-                $sSep = ', ';
-            }
-            echo "</td>";
-
-            echo "<td>";
-            $sSep = '';
-            foreach ($aRow['aAddress'] as $iWordID) {
-                echo $sSep.'#'.$aWordsIDs[$iWordID].'#';
-                $sSep = ', ';
-            }
-            echo "</td>";
-
-            echo "<td>";
-            $sSep = '';
-            foreach ($aRow['aAddressNonSearch'] as $iWordID) {
-                echo $sSep.'#'.$aWordsIDs[$iWordID].'#';
-                $sSep = ', ';
-            }
-            echo "</td>";
-
-            echo "<td>".$aRow['sCountryCode']."</td>";
-
-            echo "<td>".$aRow['sOperator']."</td>";
-            echo "<td>".$aRow['sClass']."</td>";
-            echo "<td>".$aRow['sType']."</td>";
-
-            echo "<td>".$aRow['sHouseNumber']."</td>";
-
-            echo "<td>".$aRow['fLat']."</td>";
-            echo "<td>".$aRow['fLon']."</td>";
-            echo "<td>".$aRow['fRadius']."</td>";
-
-            echo "</tr>";
         }
     }
-    echo "</table>";
+    echo '</table>';
 }
 
 
@@ -578,7 +456,7 @@ function getAddressDetails(&$oDB, $sLanguagePrefArraySQL, $iPlaceID, $sCountryCo
 {
     $sSQL = "select *,get_name_by_language(name,$sLanguagePrefArraySQL) as localname from get_addressdata($iPlaceID, $housenumber)";
     if (!$bRaw) $sSQL .= " WHERE isaddress OR type = 'country_code'";
-    $sSQL .= " order by rank_address desc,isaddress desc";
+    $sSQL .= ' order by rank_address desc,isaddress desc';
 
     $aAddressLines = chksql($oDB->getAll($sSQL));
     if ($bRaw) return $aAddressLines;
@@ -619,6 +497,81 @@ function addQuotes($s)
     return "'".$s."'";
 }
 
+function parseLatLon($sQuery)
+{
+    $sFound    = null;
+    $fQueryLat = null;
+    $fQueryLon = null;
+
+    if (preg_match('/\\s*([NS])[ ]+([0-9]+[0-9.]*)[° ]+([0-9.]+)?[′\']*[, ]+([EW])[ ]+([0-9]+)[° ]+([0-9]+[0-9.]*)[′\']*\\s*/', $sQuery, $aData)) {
+        /*               1         2                   3                    4         5            6
+         * degrees decimal minutes
+         * N 40 26.767, W 79 58.933
+         * N 40°26.767′, W 79°58.933′
+         */
+        $sFound    = $aData[0];
+        $fQueryLat = ($aData[1]=='N'?1:-1) * ($aData[2] + $aData[3]/60);
+        $fQueryLon = ($aData[4]=='E'?1:-1) * ($aData[5] + $aData[6]/60);
+    } elseif (preg_match('/\\s*([0-9]+)[° ]+([0-9]+[0-9.]*)?[′\']*[ ]+([NS])[, ]+([0-9]+)[° ]+([0-9]+[0-9.]*)?[′\' ]+([EW])\\s*/', $sQuery, $aData)) {
+        /*                     1            2                         3          4            5                      6
+         * degrees decimal minutes
+         * 40 26.767 N, 79 58.933 W
+         * 40° 26.767′ N 79° 58.933′ W
+         */
+        $sFound    = $aData[0];
+        $fQueryLat = ($aData[3]=='N'?1:-1) * ($aData[1] + $aData[2]/60);
+        $fQueryLon = ($aData[6]=='E'?1:-1) * ($aData[4] + $aData[5]/60);
+    } elseif (preg_match('/\\s*([NS])[ ]([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″"]*[, ]+([EW])[ ]([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″"]*\\s*/', $sQuery, $aData)) {
+        /*                     1        2            3              4                 5        6            7              8
+         * degrees decimal seconds
+         * N 40 26 46 W 79 58 56
+         * N 40° 26′ 46″, W 79° 58′ 56″
+         */
+        $sFound    = $aData[0];
+        $fQueryLat = ($aData[1]=='N'?1:-1) * ($aData[2] + $aData[3]/60 + $aData[4]/3600);
+        $fQueryLon = ($aData[5]=='E'?1:-1) * ($aData[6] + $aData[7]/60 + $aData[8]/3600);
+    } elseif (preg_match('/\\s*([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″" ]+([NS])[, ]+([0-9]+)[° ]+([0-9]+)[′\' ]+([0-9]+)[″" ]+([EW])\\s*/', $sQuery, $aData)) {
+        /*                     1            2              3             4          5            6              7             8
+         * degrees decimal seconds
+         * 40 26 46 N 79 58 56 W
+         * 40° 26′ 46″ N, 79° 58′ 56″ W
+         */
+        $sFound    = $aData[0];
+        $fQueryLat = ($aData[4]=='N'?1:-1) * ($aData[1] + $aData[2]/60 + $aData[3]/3600);
+        $fQueryLon = ($aData[8]=='E'?1:-1) * ($aData[5] + $aData[6]/60 + $aData[7]/3600);
+    } elseif (preg_match('/\\s*([NS])[ ]([0-9]+[0-9]*\\.[0-9]+)[°]*[, ]+([EW])[ ]([0-9]+[0-9]*\\.[0-9]+)[°]*\\s*/', $sQuery, $aData)) {
+        /*                     1        2                               3        4
+         * degrees decimal
+         * N 40.446° W 79.982°
+         */
+        $sFound    = $aData[0];
+        $fQueryLat = ($aData[1]=='N'?1:-1) * ($aData[2]);
+        $fQueryLon = ($aData[3]=='E'?1:-1) * ($aData[4]);
+    } elseif (preg_match('/\\s*([0-9]+[0-9]*\\.[0-9]+)[° ]+([NS])[, ]+([0-9]+[0-9]*\\.[0-9]+)[° ]+([EW])\\s*/', $sQuery, $aData)) {
+        /*                     1                           2          3                           4
+         * degrees decimal
+         * 40.446° N 79.982° W
+         */
+        $sFound    = $aData[0];
+        $fQueryLat = ($aData[2]=='N'?1:-1) * ($aData[1]);
+        $fQueryLon = ($aData[4]=='E'?1:-1) * ($aData[3]);
+    } elseif (preg_match('/(\\s*\\[|^\\s*|\\s*)(-?[0-9]+[0-9]*\\.[0-9]+)[, ]+(-?[0-9]+[0-9]*\\.[0-9]+)(\\]\\s*|\\s*$|\\s*)/', $sQuery, $aData)) {
+        /*                 1                   2                             3                        4
+         * degrees decimal
+         * 12.34, 56.78
+         * 12.34 56.78
+         * [12.456,-78.90]
+         */
+        $sFound    = $aData[0];
+        $fQueryLat = $aData[2];
+        $fQueryLon = $aData[3];
+    } else {
+        return false;
+    }
+
+    return array($sFound, $fQueryLat, $fQueryLon);
+}
+
 
 function geometryText2Points($geometry_as_text, $fRadius)
 {
@@ -662,3 +615,23 @@ function createPointsAroundCenter($fLon, $fLat, $fRadius)
     }
     return $aPolyPoints;
 }
+
+function closestHouseNumber($aRow)
+{
+    $fHouse = $aRow['startnumber']
+                + ($aRow['endnumber'] - $aRow['startnumber']) * $aRow['fraction'];
+
+    switch ($aRow['interpolationtype']) {
+        case 'odd':
+            $iHn = (int)($fHouse/2) * 2 + 1;
+            break;
+        case 'even':
+            $iHn = (int)(round($fHouse/2)) * 2;
+            break;
+        default:
+            $iHn = (int)(round($fHouse));
+            break;
+    }
+
+    return max(min($aRow['endnumber'], $iHn), $aRow['startnumber']);
+}

lib/log.php

@@ -20,7 +20,7 @@ function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array())
 
     $hLog = array(
              date('Y-m-d H:i:s', $aStartTime[0]).'.'.$aStartTime[1],
-             $_SERVER["REMOTE_ADDR"],
+             $_SERVER['REMOTE_ADDR'],
              $_SERVER['QUERY_STRING'],
              $sOutQuery,
              $sType,

lib/template/address-html.php

@@ -13,8 +13,10 @@
     <form class="form-inline" role="search" accept-charset="UTF-8" action="<?php echo CONST_Website_BaseURL; ?>reverse.php">
         <div class="form-group">
             <input name="format" type="hidden" value="html">
+            lat
             <input name="lat" type="text" class="form-control input-sm" placeholder="latitude" value="<?php echo $fLat; ?>" >
-            <span id="switch-coords">&lt;&gt;</span>
+            <a href="#" class="btn btn-default btn-xs" id="switch-coords" title="switch lat and lon">&lt;&gt;</a>
+            lon
             <input name="lon" type="text" class="form-control input-sm" placeholder="longitude" value="<?php echo $fLon; ?>" >
             max zoom
 

lib/template/address-json.php

@@ -12,7 +12,7 @@ if (!sizeof($aPlace))
 else
 {
     if (isset($aPlace['place_id'])) $aFilteredPlaces['place_id'] = $aPlace['place_id'];
-    $aFilteredPlaces['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright";
+    $aFilteredPlaces['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright";
     $sOSMType = formatOSMType($aPlace['osm_type']);
     if ($sOSMType)
     {

lib/template/address-jsonv2.php

@@ -12,7 +12,7 @@ if (!sizeof($aPlace))
 else
 {
     if ($aPlace['place_id']) $aFilteredPlaces['place_id'] = $aPlace['place_id'];
-    $aFilteredPlaces['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright";
+    $aFilteredPlaces['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright";
     $sOSMType = formatOSMType($aPlace['osm_type']);
     if ($sOSMType)
     {

lib/template/details-html.php

@@ -68,7 +68,8 @@
         echo '  <td class="name">'.(trim($aAddressLine['localname'])?$aAddressLine['localname']:'<span class="noname">No Name</span>')."</td>\n";
         echo '  <td>' . $aAddressLine['class'].':'.$aAddressLine['type'] . "</td>\n";
         echo '  <td>' . osmLink($aAddressLine) . "</td>\n";
-        echo '  <td>' . (isset($aAddressLine['admin_level']) ? $aAddressLine['admin_level'] : '') . "</td>\n";
+        echo '  <td>' . (isset($aAddressLine['rank_address']) ? $aAddressLine['rank_address'] : '') . "</td>\n";
+        echo '  <td>' . ($aAddressLine['admin_level'] < 15 ? $aAddressLine['admin_level'] : '') . "</td>\n";
         echo '  <td>' . format_distance($aAddressLine['distance'])."</td>\n";
         echo '  <td>' . detailsLink($aAddressLine,'details &gt;') . "</td>\n";
         echo "</tr>\n";
@@ -124,6 +125,8 @@
                         kv('Wikipedia Calculated' , wikipediaLink($aPointDetails) );
                     }
 
+                    kv('Computed Postcode', $aPointDetails['postcode']);
+                    kv('Address Tags'    , hash_to_subtable($aPointDetails['aAddressTags']) );
                     kv('Extra Tags'      , hash_to_subtable($aPointDetails['aExtraTags']) );
 
                 ?>
@@ -147,6 +150,7 @@
                       <td>Local name</td>
                       <td>Type</td>
                       <td>OSM</td>
+                      <td>Address rank</td>
                       <td>Admin level</td>
                       <td>Distance</td>
                       <td></td>

lib/template/includes/html-footer.php

@@ -3,11 +3,12 @@
         Addresses and postcodes are approximate
     </p>
     <p class="copyright">
-        &copy; <a href="http://osm.org/copyright">OpenStreetMap</a> contributors
+        &copy; <a href="https://osm.org/copyright">OpenStreetMap</a> contributors
     </p>
 </footer>
 
 <script src="js/jquery.min.js"></script>
 <script src="js/bootstrap.min.js"></script>
 <script src="js/leaflet.min.js"></script>
+<script src="js/Control.Minimap.min.js"></script>
 <script src="js/nominatim-ui.js"></script>

lib/template/includes/html-header.php

@@ -8,5 +8,6 @@
     <base href="<?php echo CONST_Website_BaseURL;?>" />
     <link href="nominatim.xml" rel="search" title="Nominatim Search" type="application/opensearchdescription+xml" />
     <link href="css/leaflet.css" rel="stylesheet" />
+    <link href="css/Control.Minimap.min.css" rel="stylesheet" />
     <link href="css/bootstrap-theme.min.css" rel="stylesheet" />
     <link href="css/bootstrap.min.css" rel="stylesheet" />

lib/template/includes/html-top-navigation.php

@@ -21,8 +21,8 @@
                     About &amp; Help <span class="caret"></span>
                 </button>
                 <ul class="dropdown-menu dropdown-menu-right">
-                    <li><a href="http://wiki.openstreetmap.org/wiki/Nominatim" target="_blank">Documentation</a></li>
+                    <li><a href="https://wiki.openstreetmap.org/wiki/Nominatim" target="_blank">Documentation</a></li>
-                    <li><a href="http://wiki.openstreetmap.org/wiki/Nominatim/FAQ" target="_blank">FAQ</a></li>
+                    <li><a href="https://wiki.openstreetmap.org/wiki/Nominatim/FAQ" target="_blank">FAQ</a></li>
                     <li role="separator" class="divider"></li>
                     <li><a href="#" class="" data-toggle="modal" data-target="#report-modal">Report problem with results</a></li>
                 </ul>

lib/template/includes/introduction.php

@@ -1,9 +1,9 @@
 <h2>Welcome to Nominatim</h2>
 
-<p>Nominatim is a search engine for <a href="http://www.openstreetmap.org">OpenStreetMap</a>
+<p>Nominatim is a search engine for <a href="https://www.openstreetmap.org">OpenStreetMap</a>
-data. This is the debugging interface. You may search for a name or address(forward search) or
+data. This is the debugging interface. You may search for a name or address (forward search) or
-look up data by its geographic coordinate(reverse search). Each result comes with a
+look up data by its geographic coordinate (reverse search). Each result comes with a
 link to a details page where you can inspect what data about the object is saved in 
 the database and investigate how the address of the object has been computed.</p>
 
-For more information visit the <a href="http://wiki.openstreetmap.org/wiki/Nominatim">Nominatim wiki page</a>.
+For more information visit the <a href="https://wiki.openstreetmap.org/wiki/Nominatim">Nominatim wiki page</a>.

lib/template/includes/report-errors.php

@@ -1,7 +1,7 @@
 <p>
-    Before reporting problems please read the <a target="_blank" href="http://wiki.openstreetmap.org/wiki/Nominatim">user documentation</a>
+    Before reporting problems please read the <a target="_blank" href="https://wiki.openstreetmap.org/wiki/Nominatim">user documentation</a>
     and
-    <a target="_blank" href="http://wiki.openstreetmap.org/wiki/Nominatim/FAQ">FAQ</a>.
+    <a target="_blank" href="https://wiki.openstreetmap.org/wiki/Nominatim/FAQ">FAQ</a>.
 
     If your problem relates to the address of a particular search result please use the 'details' link 
     to check how the address was generated before reporting a problem.
@@ -10,7 +10,7 @@
     Use <a target="_blank" href="https://github.com/openstreetmap/nominatim/issues">Nominatim issues on github</a>
     to report problems.
 <!-- You can search for existing bug reports
-    <a href="http://trac.openstreetmap.org/query?status=new&amp;status=assigned&amp;status=reopened&amp;component=nominatim&amp;order=priority">here</a>.</p>
+    <a href="https://trac.openstreetmap.org/query?status=new&amp;status=assigned&amp;status=reopened&amp;component=nominatim&amp;order=priority">here</a>.</p>
  -->
 </p>
 <p>

lib/template/search-batch-json.php

@@ -1,7 +1,7 @@
 <?php
 
 $aOutput = array();
-$aOutput['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright";
+$aOutput['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright";
 $aOutput['batch'] = array();
 
 foreach($aBatchResults as $aSearchResults)

lib/template/search-json.php

@@ -6,7 +6,7 @@ foreach($aSearchResults as $iResNum => $aPointDetails)
 {
     $aPlace = array(
             'place_id'=>$aPointDetails['place_id'],
-            'licence'=>"Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright",
+            'licence'=>"Data © OpenStreetMap contributors, ODbL 1.0. https://www.openstreetmap.org/copyright",
         );
     
     $sOSMType = formatOSMType($aPointDetails['osm_type']);

lib/template/search-jsonv2.php

@@ -5,7 +5,7 @@ foreach($aSearchResults as $iResNum => $aPointDetails)
 {
     $aPlace = array(
             'place_id'=>$aPointDetails['place_id'],
-            'licence'=>"Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright",
+            'licence'=>"Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright",
         );
 
     $sOSMType = formatOSMType($aPointDetails['osm_type']);

nominatim/export.c

@@ -152,7 +152,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
 
     pg_prepare_params[0] = PG_OID_INT8;
     res = PQprepare(conn, "placex_details",
-                    "select placex.osm_type, placex.osm_id, placex.class, placex.type, placex.name, placex.housenumber, placex.country_code, ST_AsText(placex.geometry), placex.admin_level, placex.rank_address, placex.rank_search, placex.parent_place_id, parent.osm_type, parent.osm_id, placex.indexed_status from placex left outer join placex as parent on (placex.parent_place_id = parent.place_id) where placex.place_id = $1",
+                    "select placex.osm_type, placex.osm_id, placex.class, placex.type, placex.name, placex.housenumber, placex.country_code, ST_AsText(placex.geometry), placex.admin_level, placex.rank_address, placex.rank_search, placex.parent_place_id, parent.osm_type, parent.osm_id, placex.indexed_status, placex.linked_place_id from placex left outer join placex as parent on (placex.parent_place_id = parent.place_id) where placex.place_id = $1",
                     1, pg_prepare_params);
     if (PQresultStatus(res) != PGRES_COMMAND_OK)
     {
@@ -355,7 +355,7 @@ void nominatim_exportFreeQueries(struct export_data * querySet)
 /*
  * Requirements: the prepared queries must exist
  */
-void nominatim_exportPlace(uint64_t place_id, PGconn * conn, 
+void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
   xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet)
 {
     struct export_data		querySet;
@@ -387,7 +387,7 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
         {
             // Add
             if (writer_mutex) pthread_mutex_lock( writer_mutex );
-            nominatim_exportStartMode(writer, 1);  
+            nominatim_exportStartMode(writer, 1);
         }
         else
         {
@@ -396,14 +396,14 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
             // TODO: detect changes
 
             if (writer_mutex) pthread_mutex_lock( writer_mutex );
-            nominatim_exportStartMode(writer, 2);  
+            nominatim_exportStartMode(writer, 2);
         }
     }
     else
     {
        // Add
        if (writer_mutex) pthread_mutex_lock( writer_mutex );
-       nominatim_exportStartMode(writer, 1);  
+       nominatim_exportStartMode(writer, 1);
     }
 
     xmlTextWriterStartElement(writer, BAD_CAST "feature");
@@ -417,6 +417,7 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
     xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_place_id", BAD_CAST PQgetvalue(querySet.res, 0, 11));
     xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_type", BAD_CAST PQgetvalue(querySet.res, 0, 12));
     xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_id", BAD_CAST PQgetvalue(querySet.res, 0, 13));
+    xmlTextWriterWriteAttribute(writer, BAD_CAST "linked_place_id", BAD_CAST PQgetvalue(querySet.res, 0, 15));
 
     if (PQntuples(querySet.resNames))
     {

nominatim/index.c

@@ -218,7 +218,7 @@ struct index_thread_data * thread_data, const char *structuredoutputfile)
                     usleep(1000);
 
                     // Aim for one update per second
-                    if (sleepcount++ > 500)
+                    if (sleepcount++ > 1000)
                     {
                         rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
                         if(interpolation)
@@ -438,9 +438,9 @@ void *nominatim_indexThread(void * thread_data_in)
     uint64_t    paramPlaceID;
     uint64_t    place_id;
     time_t      updateStartTime;
-    uint        table;
+    unsigned    table;
     
-    table = (uint)(thread_data->table);
+    table = thread_data->table;
 
     while (1)
     {

nominatim/index.h

@@ -14,7 +14,7 @@ struct index_thread_data
     pthread_mutex_t * count_mutex;
     xmlTextWriterPtr writer;
     pthread_mutex_t * writer_mutex;
-    uint table;
+    unsigned table;
 };
 void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile);
 void *nominatim_indexThread(void * thread_data_in);

phpcs.xml

@@ -10,8 +10,8 @@
 
   <rule ref="Generic.Files.LineLength">
     <properties>
-     <property name="lineLimit" value="199"/>
+     <property name="lineLimit" value="194"/>
-     <property name="absoluteLineLimit" value="199"/>
+     <property name="absoluteLineLimit" value="194"/>
     </properties>
   </rule>
 
@@ -129,6 +129,23 @@
 
 
 
+
+  <!-- **************************************************************
+       STRING QUOTING
+       ************************************************************** -->
+
+  <!-- Prefer single quoted strings -->
+  <rule ref="Squiz.Strings.DoubleQuoteUsage" />
+
+  <!-- We allow variabled inside double-quoted strings "abc $somevar" -->
+  <rule ref="Squiz.Strings.DoubleQuoteUsage.ContainsVar">
+    <severity>0</severity>
+  </rule>
+
+
+
+
+
   <!-- **************************************************************
        CONTROL STRUCTURES
        ************************************************************** -->

settings/defaults.php

@@ -72,7 +72,7 @@ if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true);
 //// Replication settings
 
 // Base URL of replication service
-@define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute');
+@define('CONST_Replication_Url', 'https://planet.openstreetmap.org/replication/minute');
 
 // Maximum size in MB of data to download per batch
 @define('CONST_Replication_Max_Diff_size', '30');
@@ -94,14 +94,13 @@ if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true);
 @define('CONST_Default_Lat', 20.0);
 @define('CONST_Default_Lon', 0.0);
 @define('CONST_Default_Zoom', 2);
-@define('CONST_Map_Tile_URL', 'http://{s}.tile.osm.org/{z}/{x}/{y}.png');
+@define('CONST_Map_Tile_URL', 'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png');
 @define('CONST_Map_Tile_Attribution', ''); // Set if tile source isn't osm.org
 
 @define('CONST_Search_AreaPolygons', true);
 
 @define('CONST_Search_BatchMode', false);
 
-@define('CONST_Search_TryDroppedAddressTerms', false);
 @define('CONST_Search_NameOnlySearchFrequencyThreshold', 500);
 // If set to true, then reverse order of queries will be tried by default.
 // When set to false only selected languages alloow reverse search.

sql/functions.sql

@@ -83,6 +83,26 @@ END;
 $$
 LANGUAGE plpgsql;
 
+CREATE OR REPLACE FUNCTION getorcreate_postcode_id(postcode TEXT)
+  RETURNS INTEGER
+  AS $$
+DECLARE
+  lookup_token TEXT;
+  lookup_word TEXT;
+  return_word_id INTEGER;
+BEGIN
+  lookup_word := upper(trim(postcode));
+  lookup_token := ' ' || make_standard_name(lookup_word);
+  SELECT min(word_id) FROM word WHERE word_token = lookup_token and class='place' and type='postcode' into return_word_id;
+  IF return_word_id IS NULL THEN
+    return_word_id := nextval('seq_word');
+    INSERT INTO word VALUES (return_word_id, lookup_token, lookup_word, 'place', 'postcode', null, 0);
+  END IF;
+  RETURN return_word_id;
+END;
+$$
+LANGUAGE plpgsql;
+
 CREATE OR REPLACE FUNCTION getorcreate_country(lookup_word TEXT, lookup_country_code varchar(2))
   RETURNS INTEGER
   AS $$
@@ -236,6 +256,99 @@ END;
 $$
 LANGUAGE plpgsql IMMUTABLE;
 
+
+CREATE OR REPLACE FUNCTION get_postcode_rank(country_code VARCHAR(2), postcode TEXT,
+                                      OUT rank_search SMALLINT, OUT rank_address SMALLINT)
+AS $$
+DECLARE
+  part TEXT;
+BEGIN
+    rank_search := 30;
+    rank_address := 30;
+    postcode := upper(postcode);
+
+    IF country_code = 'gb' THEN
+        IF postcode ~ '^([A-Z][A-Z]?[0-9][0-9A-Z]? [0-9][A-Z][A-Z])$' THEN
+            rank_search := 25;
+            rank_address := 5;
+        ELSEIF postcode ~ '^([A-Z][A-Z]?[0-9][0-9A-Z]? [0-9])$' THEN
+            rank_search := 23;
+            rank_address := 5;
+        ELSEIF postcode ~ '^([A-Z][A-Z]?[0-9][0-9A-Z])$' THEN
+            rank_search := 21;
+            rank_address := 5;
+        END IF;
+
+    ELSEIF country_code = 'sg' THEN
+        IF postcode ~ '^([0-9]{6})$' THEN
+            rank_search := 25;
+            rank_address := 11;
+        END IF;
+
+    ELSEIF country_code = 'de' THEN
+        IF postcode ~ '^([0-9]{5})$' THEN
+            rank_search := 21;
+            rank_address := 11;
+        END IF;
+
+    ELSE
+        -- Guess at the postcode format and coverage (!)
+        IF postcode ~ '^[A-Z0-9]{1,5}$' THEN -- Probably too short to be very local
+            rank_search := 21;
+            rank_address := 11;
+        ELSE
+            -- Does it look splitable into and area and local code?
+            part := substring(postcode from '^([- :A-Z0-9]+)([- :][A-Z0-9]+)$');
+
+            IF part IS NOT NULL THEN
+                rank_search := 25;
+                rank_address := 11;
+            ELSEIF postcode ~ '^[- :A-Z0-9]{6,}$' THEN
+                rank_search := 21;
+                rank_address := 11;
+            END IF;
+        END IF;
+    END IF;
+
+END;
+$$
+LANGUAGE plpgsql IMMUTABLE;
+
+-- Find the nearest artificial postcode for the given geometry.
+-- TODO For areas there should not be more than two inside the geometry.
+CREATE OR REPLACE FUNCTION get_nearest_postcode(country VARCHAR(2), geom GEOMETRY) RETURNS TEXT
+  AS $$
+DECLARE
+  outcode TEXT;
+  cnt INTEGER;
+BEGIN
+    -- If the geometry is an area then only one postcode must be within
+    -- that area, otherwise consider the area as not having a postcode.
+    IF ST_GeometryType(geom) in ('ST_Polygon','ST_MultiPolygon') THEN
+        SELECT min(postcode), count(*) FROM
+              (SELECT postcode FROM location_postcode
+                WHERE ST_Contains(geom, location_postcode.geometry) LIMIT 2) sub
+          INTO outcode, cnt;
+
+        IF cnt = 1 THEN
+            RETURN outcode;
+        ELSE
+            RETURN null;
+        END IF;
+    END IF;
+
+    SELECT postcode FROM location_postcode
+     WHERE ST_DWithin(geom, location_postcode.geometry, 0.05)
+          AND location_postcode.country_code = country
+     ORDER BY ST_Distance(geom, location_postcode.geometry) LIMIT 1
+    INTO outcode;
+
+    RETURN outcode;
+END;
+$$
+LANGUAGE plpgsql;
+
+
 CREATE OR REPLACE FUNCTION create_country(src HSTORE, lookup_country_code varchar(2)) RETURNS VOID
   AS $$
 DECLARE
@@ -515,36 +628,38 @@ CREATE OR REPLACE FUNCTION add_location(
     keywords INTEGER[],
     rank_search INTEGER,
     rank_address INTEGER,
+    in_postcode TEXT,
     geometry GEOMETRY
   ) 
   RETURNS BOOLEAN
   AS $$
 DECLARE
   locationid INTEGER;
-  isarea BOOLEAN;
   centroid GEOMETRY;
   diameter FLOAT;
   x BOOLEAN;
   splitGeom RECORD;
   secgeo GEOMETRY;
+  postcode TEXT;
 BEGIN
 
   IF rank_search > 25 THEN
     RAISE EXCEPTION 'Adding location with rank > 25 (% rank %)', place_id, rank_search;
   END IF;
 
---  RAISE WARNING 'Adding location with rank > 25 (% rank %)', place_id, rank_search;
-
   x := deleteLocationArea(partition, place_id, rank_search);
 
-  isarea := false;
+  -- add postcode only if it contains a single entry, i.e. ignore postcode lists
-  IF (ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(geometry)) THEN
+  postcode := NULL;
+  IF in_postcode is not null AND in_postcode not similar to '%(,|;)%' THEN
+      postcode := upper(trim (in_postcode));
+  END IF;
 
-    isArea := true;
+  IF ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') THEN
     centroid := ST_Centroid(geometry);
 
     FOR secgeo IN select split_geometry(geometry) AS geom LOOP
-      x := insertLocationAreaLarge(partition, place_id, country_code, keywords, rank_search, rank_address, false, centroid, secgeo);
+      x := insertLocationAreaLarge(partition, place_id, country_code, keywords, rank_search, rank_address, false, postcode, centroid, secgeo);
     END LOOP;
 
   ELSE
@@ -569,7 +684,7 @@ BEGIN
 --    RAISE WARNING 'adding % diameter %', place_id, diameter;
 
     secgeo := ST_Buffer(geometry, diameter);
-    x := insertLocationAreaLarge(partition, place_id, country_code, keywords, rank_search, rank_address, true, ST_Centroid(geometry), secgeo);
+    x := insertLocationAreaLarge(partition, place_id, country_code, keywords, rank_search, rank_address, true, postcode, ST_Centroid(geometry), secgeo);
 
   END IF;
 
@@ -722,54 +837,10 @@ BEGIN
             RETURN NULL;
         END IF;
 
-        NEW.postcode := NEW.address->'postcode';
+        NEW.name := hstore('ref', NEW.address->'postcode');
-        NEW.name := hstore('ref', NEW.postcode);
 
-        IF NEW.country_code = 'gb' THEN
+        SELECT * FROM get_postcode_rank(NEW.country_code, NEW.address->'postcode')
-
+          INTO NEW.rank_search, NEW.rank_address;
-          IF NEW.postcode ~ '^([A-Z][A-Z]?[0-9][0-9A-Z]? [0-9][A-Z][A-Z])$' THEN
-            NEW.rank_search := 25;
-            NEW.rank_address := 5;
-          ELSEIF NEW.postcode ~ '^([A-Z][A-Z]?[0-9][0-9A-Z]? [0-9])$' THEN
-            NEW.rank_search := 23;
-            NEW.rank_address := 5;
-          ELSEIF NEW.postcode ~ '^([A-Z][A-Z]?[0-9][0-9A-Z])$' THEN
-            NEW.rank_search := 21;
-            NEW.rank_address := 5;
-          END IF;
-
-        ELSEIF NEW.country_code = 'sg' THEN
-
-          IF NEW.postcode ~ '^([0-9]{6})$' THEN
-            NEW.rank_search := 25;
-            NEW.rank_address := 11;
-          END IF;
-
-        ELSEIF NEW.country_code = 'de' THEN
-
-          IF NEW.postcode ~ '^([0-9]{5})$' THEN
-            NEW.rank_search := 21;
-            NEW.rank_address := 11;
-          END IF;
-
-        ELSE
-          -- Guess at the postcode format and coverage (!)
-          IF upper(NEW.postcode) ~ '^[A-Z0-9]{1,5}$' THEN -- Probably too short to be very local
-            NEW.rank_search := 21;
-            NEW.rank_address := 11;
-          ELSE
-            -- Does it look splitable into and area and local code?
-            postcode := substring(upper(NEW.postcode) from '^([- :A-Z0-9]+)([- :][A-Z0-9]+)$');
-
-            IF postcode IS NOT NULL THEN
-              NEW.rank_search := 25;
-              NEW.rank_address := 11;
-            ELSEIF NEW.postcode ~ '^[- :A-Z0-9]{6,}$' THEN
-              NEW.rank_search := 21;
-              NEW.rank_address := 11;
-            END IF;
-          END IF;
-        END IF;
 
     ELSEIF NEW.class = 'place' THEN
       IF NEW.type in ('continent') THEN
@@ -844,6 +915,9 @@ BEGIN
       ELSE
         NEW.rank_address := 0;
       END IF;
+    ELSEIF NEW.class = 'leisure' and NEW.type in ('park') THEN
+      NEW.rank_search := 24;
+      NEW.rank_address := 0;
     ELSEIF NEW.class = 'natural' and NEW.type in ('peak','volcano','mountain_range') THEN
       NEW.rank_search := 18;
       NEW.rank_address := 0;
@@ -985,8 +1059,8 @@ DECLARE
   linegeo GEOMETRY;
   splitline GEOMETRY;
   sectiongeo GEOMETRY;
+  interpol_postcode TEXT;
   postcode TEXT;
-  seg_postcode TEXT;
 BEGIN
   -- deferred delete
   IF OLD.indexed_status = 100 THEN
@@ -1005,9 +1079,11 @@ BEGIN
                                                  NEW.address->'place',
                                                  NEW.partition, place_centroid, NEW.linegeo);
 
-
+  IF NEW.address is not NULL AND NEW.address ? 'postcode' AND NEW.address->'postcode' not similar to '%(,|;)%' THEN
-  IF NEW.address is not NULL and NEW.address ? 'postcode' THEN
+    interpol_postcode := NEW.address->'postcode';
-      NEW.postcode = NEW.address->'postcode';
+    housenum := getorcreate_postcode_id(NEW.address->'postcode');
+  ELSE
+    interpol_postcode := NULL;
   END IF;
 
   -- if the line was newly inserted, split the line as necessary
@@ -1020,7 +1096,6 @@ BEGIN
 
       linegeo := NEW.linegeo;
       startnumber := NULL;
-      postcode := NEW.postcode;
 
       FOR nodeidpos in 1..array_upper(waynodes, 1) LOOP
 
@@ -1053,15 +1128,24 @@ BEGIN
               sectiongeo := ST_Reverse(sectiongeo);
             END IF;
 
-            seg_postcode := coalesce(postcode,
+            -- determine postcode
-                                     prevnode.address->'postcode',
+            postcode := coalesce(interpol_postcode,
-                                     nextnode.address->'postcode');
+                                 prevnode.address->'postcode',
+                                 nextnode.address->'postcode',
+                                 postcode);
+
+            IF postcode is NULL THEN
+                SELECT placex.postcode FROM placex WHERE place_id = NEW.parent_place_id INTO postcode;
+            END IF;
+            IF postcode is NULL THEN
+                postcode := get_nearest_postcode(NEW.country_code, nextnode.geometry);
+            END IF;
 
             IF NEW.startnumber IS NULL THEN
                 NEW.startnumber := startnumber;
                 NEW.endnumber := endnumber;
                 NEW.linegeo := sectiongeo;
-                NEW.postcode := seg_postcode;
+                NEW.postcode := upper(trim(postcode));
              ELSE
               insert into location_property_osmline
                      (linegeo, partition, osm_id, parent_place_id,
@@ -1070,7 +1154,7 @@ BEGIN
                       geometry_sector, indexed_status)
               values (sectiongeo, NEW.partition, NEW.osm_id, NEW.parent_place_id,
                       startnumber, endnumber, NEW.interpolationtype,
-                      NEW.address, seg_postcode,
+                      NEW.address, postcode,
                       NEW.country_code, NEW.geometry_sector, 0);
              END IF;
           END IF;
@@ -1094,7 +1178,42 @@ END;
 $$
 LANGUAGE plpgsql;
 
+-- Trigger for updates of location_postcode
+--
+-- Computes the parent object the postcode most likely refers to.
+-- This will be the place that determines the address displayed when
+-- searching for this postcode.
+CREATE OR REPLACE FUNCTION postcode_update() RETURNS
+TRIGGER
+  AS $$
+DECLARE
+  partition SMALLINT;
+  location RECORD;
+BEGIN
+    IF NEW.indexed_status != 0 OR OLD.indexed_status = 0 THEN
+        RETURN NEW;
+    END IF;
 
+    NEW.indexed_date = now();
+
+    partition := get_partition(NEW.country_code);
+
+    SELECT * FROM get_postcode_rank(NEW.country_code, NEW.postcode)
+      INTO NEW.rank_search, NEW.rank_address;
+
+    NEW.parent_place_id = 0;
+    FOR location IN
+      SELECT place_id
+        FROM getNearFeatures(partition, NEW.geometry, NEW.rank_search, '{}'::int[])
+        WHERE NOT isguess ORDER BY rank_address DESC LIMIT 1
+    LOOP
+        NEW.parent_place_id = location.place_id;
+    END LOOP;
+
+    RETURN NEW;
+END;
+$$
+LANGUAGE plpgsql;
 
 CREATE OR REPLACE FUNCTION placex_update() RETURNS
 TRIGGER
@@ -1158,11 +1277,6 @@ BEGIN
 
   --DEBUG: RAISE WARNING 'placex_update % % (%)',NEW.osm_type,NEW.osm_id,NEW.place_id;
 
-  IF NEW.class = 'place' AND NEW.type = 'postcodearea' THEN
-    -- Silently do nothing
-    RETURN NEW;
-  END IF;
-
   NEW.indexed_date = now();
 
   result := deleteSearchName(NEW.partition, NEW.place_id);
@@ -1199,13 +1313,16 @@ BEGIN
       addr_street = NEW.address->'street';
       addr_place = NEW.address->'place';
 
-      NEW.postcode = NEW.address->'postcode';
+      IF NEW.address ? 'postcode' and NEW.address->'postcode' not similar to '%(,|;)%' THEN
+        i := getorcreate_postcode_id(NEW.address->'postcode');
+      END IF;
   END IF;
 
   -- Speed up searches - just use the centroid of the feature
   -- cheaper but less acurate
   place_centroid := ST_PointOnSurface(NEW.geometry);
   NEW.centroid := null;
+  NEW.postcode := null;
   --DEBUG: RAISE WARNING 'Computing preliminary centroid at %',ST_AsText(place_centroid);
 
   -- recalculate country and partition
@@ -1238,7 +1355,7 @@ BEGIN
                 --DEBUG: RAISE WARNING 'waterway parent %, child %/%', NEW.osm_id, i, relation_members[i];
                 FOR linked_node_id IN SELECT place_id FROM placex
                   WHERE osm_type = 'W' and osm_id = substring(relation_members[i],2,200)::bigint
-                  and class = NEW.class and type = NEW.type
+                  and class = NEW.class and type in ('river', 'stream', 'canal', 'drain', 'ditch')
                   and ( relation_members[i+1] != 'side_stream' or NEW.name->'name' = name->'name')
                 LOOP
                   UPDATE placex SET linked_place_id = NEW.place_id WHERE place_id = linked_node_id;
@@ -1442,25 +1559,16 @@ BEGIN
       NEW.country_code := location.country_code;
       --DEBUG: RAISE WARNING 'Got parent details from search name';
 
-      -- Merge the postcode into the parent's address if necessary
+      -- determine postcode
-      IF NEW.postcode IS NOT NULL THEN
+      IF NEW.rank_search > 4 THEN
-        --DEBUG: RAISE WARNING 'Merging postcode into parent';
+          IF NEW.address is not null AND NEW.address ? 'postcode' THEN
-        isin_tokens := '{}'::int[];
+              NEW.postcode = upper(trim(NEW.address->'postcode'));
-        address_street_word_id := getorcreate_word_id(make_standard_name(NEW.postcode));
+          ELSE
-        IF address_street_word_id is not null
+             SELECT postcode FROM placex WHERE place_id = NEW.parent_place_id INTO NEW.postcode;
-           and not ARRAY[address_street_word_id] <@ location.nameaddress_vector THEN
+          END IF;
-           isin_tokens := isin_tokens || address_street_word_id;
+          IF NEW.postcode is null THEN
-        END IF;
+            NEW.postcode := get_nearest_postcode(NEW.country_code, place_centroid);
-        address_street_word_id := getorcreate_name_id(make_standard_name(NEW.postcode));
+          END IF;
-        IF address_street_word_id is not null
-           and not ARRAY[address_street_word_id] <@ location.nameaddress_vector THEN
-           isin_tokens := isin_tokens || address_street_word_id;
-        END IF;
-        IF isin_tokens != '{}'::int[] THEN
-           UPDATE search_name
-              SET nameaddress_vector = search_name.nameaddress_vector || isin_tokens
-            WHERE place_id = NEW.parent_place_id;
-        END IF;
       END IF;
 
       -- If there is no name it isn't searchable, don't bother to create a search record
@@ -1478,7 +1586,7 @@ BEGIN
       -- Just be happy with inheriting from parent road only
 
       IF NEW.rank_search <= 25 and NEW.rank_address > 0 THEN
-        result := add_location(NEW.place_id, NEW.country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, NEW.geometry);
+        result := add_location(NEW.place_id, NEW.country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, upper(trim(NEW.address->'postcode')), NEW.geometry);
         --DEBUG: RAISE WARNING 'Place added to location table';
       END IF;
 
@@ -1649,7 +1757,7 @@ BEGIN
   END IF;
 
   -- make sure all names are in the word table
-  IF NEW.admin_level = 2 AND NEW.class = 'boundary' AND NEW.type = 'administrative' AND NEW.country_code IS NOT NULL THEN
+  IF NEW.admin_level = 2 AND NEW.class = 'boundary' AND NEW.type = 'administrative' AND NEW.country_code IS NOT NULL AND NEW.osm_type = 'R' THEN
     perform create_country(NEW.name, lower(NEW.country_code));
     --DEBUG: RAISE WARNING 'Country names updated';
   END IF;
@@ -1665,6 +1773,7 @@ BEGIN
     isin := avals(NEW.address);
     IF array_upper(isin, 1) IS NOT NULL THEN
       FOR i IN 1..array_upper(isin, 1) LOOP
+        -- TODO further split terms with comma and semicolon
         address_street_word_id := get_name_id(make_standard_name(isin[i]));
         IF address_street_word_id IS NOT NULL AND NOT(ARRAY[address_street_word_id] <@ isin_tokens) THEN
           nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
@@ -1679,26 +1788,6 @@ BEGIN
       END LOOP;
     END IF;
   END IF;
-  --DEBUG: RAISE WARNING '"address:* tokens collected';
-  IF NEW.postcode IS NOT NULL THEN
-    isin := regexp_split_to_array(NEW.postcode, E'[;,]');
-    IF array_upper(isin, 1) IS NOT NULL THEN
-      FOR i IN 1..array_upper(isin, 1) LOOP
-        address_street_word_id := get_name_id(make_standard_name(isin[i]));
-        IF address_street_word_id IS NOT NULL AND NOT(ARRAY[address_street_word_id] <@ isin_tokens) THEN
-          nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
-          isin_tokens := isin_tokens || address_street_word_id;
-        END IF;
-
-        -- merge into address vector
-        address_street_word_id := get_word_id(make_standard_name(isin[i]));
-        IF address_street_word_id IS NOT NULL THEN
-          nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
-        END IF;
-      END LOOP;
-    END IF;
-  END IF;
-  --DEBUG: RAISE WARNING 'postcode tokens collected';
 
   -- %NOTIGERDATA% IF 0 THEN
   -- for the USA we have an additional address table.  Merge in zip codes from there too
@@ -1770,6 +1859,11 @@ BEGIN
         VALUES (NEW.place_id, location.place_id, true, location_isaddress, location.distance, location.rank_address);
 
       IF location_isaddress THEN
+        -- add postcode if we have one
+        -- (If multiple postcodes are available, we end up with the highest ranking one.)
+        IF location.postcode is not null THEN
+            NEW.postcode = location.postcode;
+        END IF;
 
         address_havelevel[location.rank_address] := true;
         IF NOT location.isguess THEN
@@ -1804,6 +1898,11 @@ BEGIN
               nameaddress_vector := array_merge(nameaddress_vector, location.keywords::integer[]);
               INSERT INTO place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address)
                 VALUES (NEW.place_id, location.place_id, false, NOT address_havelevel[location.rank_address], location.distance, location.rank_address);
+              IF NEW.postcode is null AND location.postcode is not null
+                 AND NOT address_havelevel[location.rank_address] THEN
+                NEW.postcode := location.postcode;
+              END IF;
+
               address_havelevel[location.rank_address] := true;
 
               IF location.rank_address > parent_place_id_rank THEN
@@ -1846,11 +1945,20 @@ BEGIN
   END IF;
   --DEBUG: RAISE WARNING 'search terms for long ways added';
 
+  IF NEW.address is not null AND NEW.address ? 'postcode' 
+     AND NEW.address->'postcode' not similar to '%(,|;)%' THEN
+    NEW.postcode := upper(trim(NEW.address->'postcode'));
+  END IF;
+
+  IF NEW.postcode is null AND NEW.rank_search > 8 THEN
+    NEW.postcode := get_nearest_postcode(NEW.country_code, NEW.geometry);
+  END IF;
+
   -- if we have a name add this to the name search table
   IF NEW.name IS NOT NULL THEN
 
     IF NEW.rank_search <= 25 and NEW.rank_address > 0 THEN
-      result := add_location(NEW.place_id, NEW.country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, NEW.geometry);
+      result := add_location(NEW.place_id, NEW.country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, upper(trim(NEW.address->'postcode')), NEW.geometry);
       --DEBUG: RAISE WARNING 'added to location (full)';
     END IF;
 
@@ -2247,46 +2355,6 @@ END;
 $$
 LANGUAGE plpgsql IMMUTABLE;
 
-
-CREATE OR REPLACE FUNCTION get_address_postcode(for_place_id BIGINT) RETURNS TEXT
-  AS $$
-DECLARE
-  result TEXT[];
-  search TEXT[];
-  for_postcode TEXT;
-  found INTEGER;
-  location RECORD;
-BEGIN
-
-  found := 1000;
-  search := ARRAY['ref'];
-  result := '{}';
-
-  select postcode from placex where place_id = for_place_id limit 1 into for_postcode;
-
-  FOR location IN 
-    select rank_address,name,distance,length(name::text) as namelength 
-      from place_addressline join placex on (address_place_id = placex.place_id) 
-      where place_addressline.place_id = for_place_id and rank_address in (5,11)
-      order by rank_address desc,rank_search desc,fromarea desc,distance asc,namelength desc
-  LOOP
-    IF array_upper(search, 1) IS NOT NULL AND array_upper(location.name, 1) IS NOT NULL THEN
-      FOR j IN 1..array_upper(search, 1) LOOP
-        FOR k IN 1..array_upper(location.name, 1) LOOP
-          IF (found > location.rank_address AND location.name[k].key = search[j] AND location.name[k].value != '') AND NOT result @> ARRAY[trim(location.name[k].value)] AND (for_postcode IS NULL OR location.name[k].value ilike for_postcode||'%') THEN
-            result[(100 - location.rank_address)] := trim(location.name[k].value);
-            found := location.rank_address;
-          END IF;
-        END LOOP;
-      END LOOP;
-    END IF;
-  END LOOP;
-
-  RETURN array_to_string(result,', ');
-END;
-$$
-LANGUAGE plpgsql;
-
 --housenumber only needed for tiger data
 CREATE OR REPLACE FUNCTION get_address_by_language(for_place_id BIGINT, housenumber INTEGER, languagepref TEXT[]) RETURNS TEXT
   AS $$
@@ -2370,11 +2438,19 @@ BEGIN
   -- %NOAUXDATA% IF 0 THEN
   IF for_place_id IS NULL THEN
     select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux
-      WHERE place_id = in_place_id 
+      WHERE place_id = in_place_id
       INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
   END IF;
   -- %NOAUXDATA% END IF;
 
+  -- postcode table
+  IF for_place_id IS NULL THEN
+    select parent_place_id, country_code, rank_address, postcode, 'place', 'postcode'
+      FROM location_postcode
+      WHERE place_id = in_place_id
+      INTO for_place_id, searchcountrycode, searchrankaddress, searchpostcode, searchclass, searchtype;
+  END IF;
+
   IF for_place_id IS NULL THEN
     select parent_place_id, country_code, housenumber, rank_search, postcode, name, class, type from placex 
       WHERE place_id = in_place_id and  rank_search > 27
@@ -2393,9 +2469,8 @@ BEGIN
   found := 1000;
   hadcountry := false;
   FOR location IN 
-    select placex.place_id, osm_type, osm_id,
+    select placex.place_id, osm_type, osm_id, name,
-      CASE WHEN class = 'place' and type = 'postcode' THEN hstore('name', postcode) ELSE name END as name,
+      class, type, admin_level, true as isaddress,
-      class, type, admin_level, true as fromarea, true as isaddress,
       CASE WHEN rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address,
       0 as distance, country_code, postcode
       from placex
@@ -2405,13 +2480,9 @@ BEGIN
     IF searchcountrycode IS NULL AND location.country_code IS NOT NULL THEN
       searchcountrycode := location.country_code;
     END IF;
-    IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN
+    IF location.type in ('postcode', 'postal_code') THEN
       location.isaddress := FALSE;
-    END IF;
+    ELSEIF location.rank_address = 4 THEN
-    IF searchpostcode IS NULL and location.postcode IS NOT NULL THEN
-      searchpostcode := location.postcode;
-    END IF;
-    IF location.rank_address = 4 AND location.isaddress THEN
       hadcountry := true;
     END IF;
     IF location.rank_address < 4 AND NOT hadcountry THEN
@@ -2422,15 +2493,14 @@ BEGIN
       END IF;
     END IF;
     countrylocation := ROW(location.place_id, location.osm_type, location.osm_id, location.name, location.class, 
-                           location.type, location.admin_level, location.fromarea, location.isaddress, location.rank_address, 
+                           location.type, location.admin_level, true, location.isaddress, location.rank_address,
                            location.distance)::addressline;
     RETURN NEXT countrylocation;
     found := location.rank_address;
   END LOOP;
 
   FOR location IN 
-    select placex.place_id, osm_type, osm_id,
+    select placex.place_id, osm_type, osm_id, name,
-      CASE WHEN class = 'place' and type = 'postcode' THEN hstore('name', postcode) ELSE name END as name,
       CASE WHEN extratags ? 'place' THEN 'place' ELSE class END as class,
       CASE WHEN extratags ? 'place' THEN extratags->'place' ELSE type END as type,
       admin_level, fromarea, isaddress,
@@ -2439,7 +2509,7 @@ BEGIN
       from place_addressline join placex on (address_place_id = placex.place_id) 
       where place_addressline.place_id = for_place_id 
       and (cached_rank_address > 0 AND cached_rank_address < searchrankaddress)
-      and address_place_id != for_place_id
+      and address_place_id != for_place_id and linked_place_id is null
       and (placex.country_code IS NULL OR searchcountrycode IS NULL OR placex.country_code = searchcountrycode)
       order by rank_address desc,isaddress desc,fromarea desc,distance asc,rank_search desc
   LOOP
@@ -2447,12 +2517,9 @@ BEGIN
     IF searchcountrycode IS NULL AND location.country_code IS NOT NULL THEN
       searchcountrycode := location.country_code;
     END IF;
-    IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN
+    IF location.type in ('postcode', 'postal_code') THEN
       location.isaddress := FALSE;
     END IF;
-    IF searchpostcode IS NULL and location.isaddress and location.type != 'postcode' and location.postcode IS NOT NULL and location.postcode not similar to '%(,|;)%' THEN
-      searchpostcode := location.postcode;
-    END IF;
     IF location.rank_address = 4 AND location.isaddress THEN
       hadcountry := true;
     END IF;
@@ -2486,7 +2553,6 @@ BEGIN
 
   IF searchhousename IS NOT NULL THEN
     location := ROW(in_place_id, null, null, searchhousename, searchclass, searchtype, null, true, true, 29, 0)::addressline;
---    location := ROW(in_place_id, null, null, searchhousename, 'place', 'house_name', null, true, true, 29, 0)::addressline;
     RETURN NEXT location;
   END IF;
 
@@ -2632,9 +2698,7 @@ BEGIN
   IF out_postcode IS NULL THEN
     SELECT postcode from placex where place_id = out_parent_place_id INTO out_postcode;
   END IF;
-  IF out_postcode IS NULL THEN
+  -- XXX look into postcode table
-    out_postcode := getNearestPostcode(out_partition, place_centroid);
-  END IF;
 
   newpoints := 0;
   insert into location_property_aux (place_id, partition, parent_place_id, housenumber, postcode, centroid)
@@ -2892,7 +2956,7 @@ BEGIN
     IF ST_GeometryType(placegeom) in ('ST_Polygon','ST_MultiPolygon') THEN
       FOR geom IN select split_geometry(placegeom) FROM placex WHERE place_id = placeid LOOP
         update placex set indexed_status = 2 where (st_covers(geom, placex.geometry) OR ST_Intersects(geom, placex.geometry)) 
-        AND rank_search > rank and indexed_status = 0 and ST_geometrytype(placex.geometry) = 'ST_Point' and (rank_search < 28 or name is not null or (rank >= 16 and address > 'place'));
+        AND rank_search > rank and indexed_status = 0 and ST_geometrytype(placex.geometry) = 'ST_Point' and (rank_search < 28 or name is not null or (rank >= 16 and address ? 'place'));
         update placex set indexed_status = 2 where (st_covers(geom, placex.geometry) OR ST_Intersects(geom, placex.geometry)) 
         AND rank_search > rank and indexed_status = 0 and ST_geometrytype(placex.geometry) != 'ST_Point' and (rank_search < 28 or name is not null or (rank >= 16 and address ? 'place'));
       END LOOP;

sql/indices.src.sql

@@ -25,4 +25,5 @@ DROP INDEX IF EXISTS place_id_idx;
 CREATE UNIQUE INDEX idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index};
 
 
-CREATE INDEX idx_gb_postcode_postcode ON gb_postcode USING BTREE (postcode) {ts:search-index};
+CREATE UNIQUE INDEX idx_postcode_id ON location_postcode USING BTREE (place_id) {ts:search-index};
+CREATE INDEX idx_postcode_postcode ON location_postcode USING BTREE (postcode) {ts:search-index};

sql/loaddata.sql

@@ -1,16 +0,0 @@
-TRUNCATE placex;
-TRUNCATE search_name;
-TRUNCATE place_addressline;
-TRUNCATE location_area;
-
-DROP SEQUENCE seq_place;
-CREATE SEQUENCE seq_place start 100000;
-
-insert into placex (osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, country_code, extratags, geometry)
-             select osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, country_code, extratags, geometry from place where osm_type = 'N';
-insert into placex (osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, country_code, extratags, geometry)
-             select osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, country_code, extratags, geometry from place where osm_type = 'W';
-insert into placex (osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, country_code, extratags, geometry)
-             select osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, country_code, extratags, geometry from place where osm_type = 'R';
-
---select count(*) from (select create_interpolation(osm_id, housenumber) from placex where indexed=false and class='place' and type='houses') as x;

sql/partition-functions.src.sql

@@ -6,12 +6,12 @@ BEGIN
 -- start
   IF in_partition = -partition- THEN
     FOR r IN 
-      SELECT place_id, keywords, rank_address, rank_search, min(ST_Distance(feature, centroid)) as distance, isguess, centroid FROM (
+      SELECT place_id, keywords, rank_address, rank_search, min(ST_Distance(feature, centroid)) as distance, isguess, postcode, centroid FROM (
         SELECT * FROM location_area_large_-partition- WHERE ST_Intersects(geometry, feature) and rank_search < maxrank
         UNION ALL
         SELECT * FROM location_area_country WHERE ST_Intersects(geometry, feature) and rank_search < maxrank
       ) as location_area
-      GROUP BY place_id, keywords, rank_address, rank_search, isguess, centroid
+      GROUP BY place_id, keywords, rank_address, rank_search, isguess, postcode, centroid
       ORDER BY rank_address, isin_tokens && keywords desc, isguess asc,
         ST_Distance(feature, centroid) *
           CASE 
@@ -55,8 +55,8 @@ $$
 LANGUAGE plpgsql;
 
 create or replace function insertLocationAreaLarge(
-  in_partition INTEGER, in_place_id BIGINT, in_country_code VARCHAR(2), in_keywords INTEGER[], 
+  in_partition INTEGER, in_place_id BIGINT, in_country_code VARCHAR(2), in_keywords INTEGER[],
-  in_rank_search INTEGER, in_rank_address INTEGER, in_estimate BOOLEAN, 
+  in_rank_search INTEGER, in_rank_address INTEGER, in_estimate BOOLEAN, postcode TEXT,
   in_centroid GEOMETRY, in_geometry GEOMETRY) RETURNS BOOLEAN AS $$
 DECLARE
 BEGIN
@@ -72,8 +72,8 @@ BEGIN
 
 -- start
   IF in_partition = -partition- THEN
-    INSERT INTO location_area_large_-partition- (partition, place_id, country_code, keywords, rank_search, rank_address, isguess, centroid, geometry)
+    INSERT INTO location_area_large_-partition- (partition, place_id, country_code, keywords, rank_search, rank_address, isguess, postcode, centroid, geometry)
-      values (in_partition, in_place_id, in_country_code, in_keywords, in_rank_search, in_rank_address, in_estimate, in_centroid, in_geometry);
+      values (in_partition, in_place_id, in_country_code, in_keywords, in_rank_search, in_rank_address, in_estimate, postcode, in_centroid, in_geometry);
     RETURN TRUE;
   END IF;
 -- end
@@ -173,29 +173,6 @@ $$
 LANGUAGE plpgsql;
 
 
-create or replace function getNearestPostcode(in_partition INTEGER, point GEOMETRY) 
-  RETURNS TEXT AS $$
-DECLARE
-  out_postcode TEXT;
-BEGIN
-
--- start
-  IF in_partition = -partition- THEN
-    SELECT postcode
-        FROM location_area_large_-partition- join placex using (place_id)
-        WHERE st_contains(location_area_large_-partition-.geometry, point)
-        AND class = 'place' and type = 'postcode' 
-      ORDER BY st_distance(location_area_large_-partition-.centroid, point) ASC limit 1
-      INTO out_postcode;
-    RETURN out_postcode;
-  END IF;
--- end
-
-  RAISE EXCEPTION 'Unknown partition %', in_partition;
-END
-$$
-LANGUAGE plpgsql;
-
 create or replace function insertSearchName(
   in_partition INTEGER, in_place_id BIGINT, in_country_code VARCHAR(2), 
   in_name_vector INTEGER[], in_nameaddress_vector INTEGER[],

sql/partition-tables.src.sql

@@ -21,6 +21,7 @@ create type nearfeaturecentr as (
   rank_search smallint,
   distance float,
   isguess boolean,
+  postcode TEXT,
   centroid GEOMETRY
 );
 
@@ -29,9 +30,9 @@ CREATE TABLE search_name_blank (
   place_id BIGINT,
   search_rank smallint,
   address_rank smallint,
-  name_vector integer[]
+  name_vector integer[],
+  centroid GEOMETRY(Geometry, 4326)
   );
-SELECT AddGeometryColumn('search_name_blank', 'centroid', 4326, 'GEOMETRY', 2);
 
 
 CREATE TABLE location_area_country () INHERITS (location_area_large) {ts:address-data};
@@ -54,9 +55,9 @@ CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition-
 CREATE TABLE location_road_-partition- (
   place_id BIGINT,
   partition SMALLINT,
-  country_code VARCHAR(2)
+  country_code VARCHAR(2),
+  geometry GEOMETRY(Geometry, 4326)
   ) {ts:address-data};
-SELECT AddGeometryColumn('location_road_-partition-', 'geometry', 4326, 'GEOMETRY', 2);
 CREATE INDEX idx_location_road_-partition-_geometry ON location_road_-partition- USING GIST (geometry) {ts:address-index};
 CREATE INDEX idx_location_road_-partition-_place_id ON location_road_-partition- USING BTREE (place_id) {ts:address-index};
 

sql/tables.sql

@@ -61,10 +61,11 @@ CREATE TABLE location_area (
   rank_search SMALLINT NOT NULL,
   rank_address SMALLINT NOT NULL,
   country_code VARCHAR(2),
-  isguess BOOL
+  isguess BOOL,
+  postcode TEXT,
+  centroid GEOMETRY(Point, 4326),
+  geometry GEOMETRY(Geometry, 4326)
   );
-SELECT AddGeometryColumn('location_area', 'centroid', 4326, 'POINT', 2);
-SELECT AddGeometryColumn('location_area', 'geometry', 4326, 'GEOMETRY', 2);
 
 CREATE TABLE location_area_large () INHERITS (location_area);
 
@@ -74,9 +75,9 @@ CREATE TABLE location_property (
   parent_place_id BIGINT,
   partition SMALLINT,
   housenumber TEXT,
-  postcode TEXT
+  postcode TEXT,
+  centroid GEOMETRY(Point, 4326)
   );
-SELECT AddGeometryColumn('location_property', 'centroid', 4326, 'POINT', 2);
 
 CREATE TABLE location_property_aux () INHERITS (location_property);
 CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
@@ -125,9 +126,9 @@ CREATE TABLE search_name (
   address_rank SMALLINT,
   name_vector integer[],
   nameaddress_vector integer[],
-  country_code varchar(2)
+  country_code varchar(2),
+  centroid GEOMETRY(Geometry, 4326)
   ) {ts:search-data};
-SELECT AddGeometryColumn('search_name', 'centroid', 4326, 'GEOMETRY', 2);
 CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id) {ts:search-index};
 
 drop table IF EXISTS place_addressline;
@@ -157,9 +158,9 @@ CREATE TABLE placex (
   wikipedia TEXT, -- calculated wikipedia article name (language:title)
   country_code varchar(2),
   housenumber TEXT,
-  postcode TEXT
+  postcode TEXT,
+  centroid GEOMETRY(Geometry, 4326)
   ) {ts:search-data};
-SELECT AddGeometryColumn('placex', 'centroid', 4326, 'GEOMETRY', 2);
 CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {ts:search-index};
 CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {ts:search-index};
 CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {ts:address-index} WHERE linked_place_id IS NOT NULL;
@@ -197,8 +198,24 @@ CREATE TRIGGER place_before_delete BEFORE DELETE ON place
 CREATE TRIGGER place_before_insert BEFORE INSERT ON place
     FOR EACH ROW EXECUTE PROCEDURE place_insert();
 
-DROP SEQUENCE IF EXISTS seq_postcodes;
+-- Table for synthetic postcodes.
-CREATE SEQUENCE seq_postcodes start 1;
+DROP TABLE IF EXISTS location_postcode;
+CREATE TABLE location_postcode (
+  place_id BIGINT,
+  parent_place_id BIGINT,
+  rank_search SMALLINT,
+  rank_address SMALLINT,
+  indexed_status SMALLINT,
+  indexed_date TIMESTAMP,
+  country_code varchar(2),
+  postcode TEXT,
+  geometry GEOMETRY(Geometry, 4326)
+  );
+CREATE INDEX idx_postcode_geometry ON location_postcode USING GIST (geometry) {ts:address-index};
+GRANT SELECT ON location_postcode TO "{www-user}" ;
+
+CREATE TRIGGER location_postcode_before_update BEFORE UPDATE ON location_postcode
+    FOR EACH ROW EXECUTE PROCEDURE postcode_update();
 
 DROP TABLE IF EXISTS import_polygon_error;
 CREATE TABLE import_polygon_error (
@@ -209,10 +226,10 @@ CREATE TABLE import_polygon_error (
   name HSTORE,
   country_code varchar(2),
   updated timestamp,
-  errormessage text
+  errormessage text,
+  prevgeometry GEOMETRY(Geometry, 4326),
+  newgeometry GEOMETRY(Geometry, 4326)
   );
-SELECT AddGeometryColumn('import_polygon_error', 'prevgeometry', 4326, 'GEOMETRY', 2);
-SELECT AddGeometryColumn('import_polygon_error', 'newgeometry', 4326, 'GEOMETRY', 2);
 CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id);
 GRANT SELECT ON import_polygon_error TO "{www-user}";
 

sql/update-postcodes.sql

@@ -0,0 +1,52 @@
+-- Create a temporary table with postcodes from placex.
+
+CREATE TEMP TABLE tmp_new_postcode_locations AS
+SELECT country_code,
+       upper(trim (both ' ' from address->'postcode')) as pc,
+       ST_Centroid(ST_Collect(ST_Centroid(geometry))) as centroid
+  FROM placex
+ WHERE address ? 'postcode'
+       AND address->'postcode' NOT SIMILAR TO '%(,|;)%'
+       AND geometry IS NOT null
+GROUP BY country_code, pc;
+
+CREATE INDEX idx_tmp_new_postcode_locations
+          ON tmp_new_postcode_locations (pc, country_code);
+
+-- add extra US postcodes
+INSERT INTO tmp_new_postcode_locations (country_code, pc, centroid)
+    SELECT 'us', postcode, ST_SetSRID(ST_Point(x,y),4326)
+      FROM us_postcode u
+      WHERE NOT EXISTS (SELECT 0 FROM tmp_new_postcode_locations new
+                         WHERE new.country_code = 'us' AND new.pc = u.postcode);
+-- add extra UK postcodes
+INSERT INTO tmp_new_postcode_locations (country_code, pc, centroid)
+    SELECT 'gb', postcode, geometry FROM gb_postcode g
+     WHERE NOT EXISTS (SELECT 0 FROM tmp_new_postcode_locations new
+                             WHERE new.country_code = 'gb' and new.pc = g.postcode);
+
+-- Remove all postcodes that are no longer valid
+DELETE FROM location_postcode old
+  WHERE NOT EXISTS(SELECT 0 FROM tmp_new_postcode_locations new
+                   WHERE old.postcode = new.pc
+                         AND old.country_code = new.country_code);
+
+-- Update geometries where necessary
+UPDATE location_postcode old SET geometry = new.centroid, indexed_status = 1
+  FROM tmp_new_postcode_locations new
+ WHERE old.postcode = new.pc AND old.country_code = new.country_code
+       AND ST_AsText(old.geometry) != ST_AsText(new.centroid);
+
+-- Remove all postcodes that already exist from the temporary table
+DELETE FROM tmp_new_postcode_locations new
+    WHERE EXISTS(SELECT 0 FROM location_postcode old
+                 WHERE old.postcode = new.pc AND old.country_code = new.country_code);
+
+-- Add newly added postcode
+INSERT INTO location_postcode
+  (place_id, indexed_status, country_code, postcode, geometry)
+  SELECT nextval('seq_place'), 1, country_code, pc, centroid
+    FROM tmp_new_postcode_locations new;
+
+-- Finally index the newly inserted postcodes
+UPDATE location_postcode SET indexed_status = 0 WHERE indexed_status > 0;

sql/words.sql

@@ -4,7 +4,7 @@ CREATE TABLE word_frequencies AS
     WHERE v is not null
  GROUP BY id);
 
-select count(make_keywords(v)) from (select distinct address->'postcode' as v from place where address ? 'postcode') as w where v is not null;
+select count(getorcreate_postcode_id(v)) from (select distinct address->'postcode' as v from place where address ? 'postcode') as w where v is not null;
 select count(getorcreate_housenumber_id(make_standard_name(v))) from (select distinct address->'housenumber' as v from place where address ? 'housenumber') as w;
 
 -- copy the word frequencies

sql/words_from_search_name.sql

@@ -0,0 +1,11 @@
+DROP TABLE IF EXISTS word_frequencies;
+CREATE TABLE word_frequencies AS
+ SELECT unnest(name_vector) as id, count(*) FROM search_name GROUP BY id;
+
+CREATE INDEX idx_word_frequencies ON word_frequencies(id);
+
+UPDATE word SET search_name_count = count
+  FROM word_frequencies
+ WHERE word_token like ' %' and word_id = id;
+
+DROP TABLE word_frequencies;

test/README.md

@@ -100,7 +100,7 @@ be documented.
 These tests are meant to test the different API endpoints and their parameters.
 They require a preimported test database, which consists of the import of a
 planet extract. A precompiled PBF with the necessary data can be downloaded from
-http://www.nominatim.org/data/test/nominatim-api-testdata.pbf
+https://www.nominatim.org/data/test/nominatim-api-testdata.pbf
 
 The polygons defining the extract can be found in the test/testdb
 directory. There is also a reduced set of wikipedia data for this extract,
@@ -112,7 +112,7 @@ planets are likely to work as well but you may see isolated test
 failures where the data has changed. To recreate the input data
 for the test database run:
 
-    wget http://free.nchc.org.tw/osm.planet/pbf/planet-160725.osm.pbf
+    wget https://free.nchc.org.tw/osm.planet/pbf/planet-160725.osm.pbf
     osmconvert planet-160725.osm.pbf -B=test/testdb/testdb.polys -o=testdb.pbf
 
 Before importing make sure to add the following to your local settings:
@@ -123,15 +123,17 @@ Before importing make sure to add the following to your local settings:
 #### Code Coverage
 
 The API tests also support code coverage tests. You need to install
-PHP_CodeCoverage. On Debian/Ubuntu run:
+[PHP_CodeCoverage](https://github.com/sebastianbergmann/php-code-coverage).
+On Debian/Ubuntu run:
 
-    apt-get install php-codecoverage
+    apt-get install php-codecoverage php-xdebug
 
 The run the API tests as follows:
 
     behave api -DPHPCOV=<coverage output dir>
 
-To generate reports, you can use the phpcov tool:
+The output directory must be an absolute path. To generate reports, you can use
+the [phpcov](https://github.com/sebastianbergmann/phpcov) tool:
 
     phpcov merge --html=<report output dir> <coverage output dir>
 

test/bdd/api/details/simple.feature

@@ -12,3 +12,9 @@ Feature: Object details
      | N4267356889 |
      | W230804120 |
      | R123924 |
+
+    Scenario: Details with keywords
+        When sending details query for W78099902
+            | keywords |
+            | 1 |
+        Then the result is valid html

test/bdd/api/reverse/params.feature

@@ -15,6 +15,15 @@ Feature: Parameters for Reverse API
       | jsonv2 |
       | xml |
 
+    Scenario Outline: Coordinates must be floating-point numbers
+        When sending reverse coordinates <coords>
+        Then a HTTP 400 is returned
+
+    Examples:
+      | coords    |
+      | -45.3,;   |
+      | gkjd,50   |
+
     Scenario Outline: Reverse Geocoding with extratags
         When sending <format> reverse coordinates 10.776234290950017,106.70425325632095
           | extratags |

test/bdd/api/reverse/queries.feature

@@ -23,3 +23,31 @@ Feature: Reverse geocoding
         And result addresses contain
           | road            | postcode | country_code |
           | West 1st Street | 57274    | us |
+
+    Scenario: Interpolated house number
+        When sending jsonv2 reverse coordinates -33.231795578514635,-54.38682173844428
+        Then results contain
+          | osm_type | category | type |
+          | way      | place    | house |
+        And result addresses contain
+          | house_number | road |
+          | 1410         | Juan Antonio Lavalleja |
+
+    Scenario: Address with non-numerical house number
+        When sending jsonv2 reverse coordinates 53.579805460944,9.9475670458196
+        Then result addresses contain
+          | house_number | road |
+          | 43 Haus 4    | Stellinger Weg |
+
+
+    Scenario: Address with numerical house number
+        When sending jsonv2 reverse coordinates 53.580206752486,9.9502944945198
+        Then result addresses contain
+          | house_number | road |
+          | 5            | Clasingstraße |
+
+    Scenario: Location off the coast
+        When sending jsonv2 reverse coordinates 54.046489113,8.5546870529
+        Then results contain
+         | display_name |
+         | Freie und Hansestadt Hamburg, Deutschland |

test/bdd/api/search/params.feature

@@ -19,6 +19,12 @@ Feature: Search queries
         And result 0 has not attributes address
         And result 0 has bounding box in 46.5,47.5,9,10
 
+    Scenario: Unknown formats returns a user error
+        When sending search query "Vaduz"
+          | format |
+          | x45    |
+        Then a HTTP 400 is returned
+
     Scenario: JSON search with addressdetails
         When sending json search query "Montevideo" with address
         Then address of result 0 is
@@ -66,15 +72,21 @@ Feature: Search queries
         Then there are duplicates
 
     Scenario: Search with bounded viewbox in right area
-        When sending json search query "restaurant" with address
+        When sending json search query "bar" with address
           | bounded | viewbox |
           | 1       | -56.16786,-34.84061,-56.12525,-34.86526 |
         Then result addresses contain
           | city |
           | Montevideo |
 
+    Scenario: Country search with bounded viewbox remain in the area
+        When sending json search query "" with address
+          | bounded | viewbox                                 | country |
+          | 1       | -56.16786,-34.84061,-56.12525,-34.86526 | de |
+        Then less than 1 result is returned
+
     Scenario: Search with bounded viewboxlbrt in right area
-        When sending json search query "restaurant" with address
+        When sending json search query "bar" with address
           | bounded | viewboxlbrt |
           | 1       | -56.16786,-34.86526,-56.12525,-34.84061 |
         Then result addresses contain
@@ -90,7 +102,7 @@ Feature: Search queries
           | ^[^,]*[Rr]estaurant.* |
 
     Scenario: bounded search remains within viewbox, even with no results
-         When sending json search query "restaurant"
+         When sending json search query "[restaurant]"
            | bounded | viewbox |
            | 1       | 43.5403125,-5.6563282,43.54285,-5.662003 |
         Then less than 1 result is returned
@@ -115,6 +127,38 @@ Feature: Search queries
           | ID | state |
           | 0  | Florida |
 
+    Scenario: viewboxes cannot be points
+        When sending json search query "foo"
+          | viewbox |
+          | 1.01,34.6,1.01,34.6 |
+        Then a HTTP 400 is returned
+
+    Scenario Outline: viewbox must have four coordinate numbers
+        When sending json search query "foo"
+          | viewbox |
+          | <viewbox> |
+        Then a HTTP 400 is returned
+
+    Examples:
+        | viewbox |
+        | 34      |
+        | 0.003,-84.4 |
+        | 5.2,4.5542,12.4 |
+        | 23.1,-6,0.11,44.2,9.1 |
+
+    Scenario Outline: viewboxlbrt must have four coordinate numbers
+        When sending json search query "foo"
+          | viewboxlbrt |
+          | <viewbox> |
+        Then a HTTP 400 is returned
+
+    Examples:
+        | viewbox |
+        | 34      |
+        | 0.003,-84.4 |
+        | 5.2,4.5542,12.4 |
+        | 23.1,-6,0.11,44.2,9.1 |
+
     Scenario: Overly large limit number for search results
         When sending json search query "restaurant"
           | limit |
@@ -127,6 +171,12 @@ Feature: Search queries
           | 4 |
         Then exactly 4 results are returned
 
+    Scenario: Limit parameter must be a number
+        When sending search query "Blue Laguna"
+          | limit |
+          | );    |
+        Then a HTTP 400 is returned
+
     Scenario: Restrict to feature type country
         When sending xml search query "Uruguay"
         Then results contain
@@ -298,3 +348,11 @@ Feature: Search queries
         | xml      | geojson |
         | json     | geojson |
         | jsonv2   | geojson |
+
+    Scenario: Search along a route
+        When sending json search query "restaurant" with address
+          | bounded | routewidth | route                                   |
+          | 1       | 0.1        | -103.23255,44.08198,-103.22516,44.08079 |
+        Then result addresses contain
+          | city |
+          | Rapid City |

test/bdd/api/search/postcode.feature

@@ -0,0 +1,35 @@
+@APIDB
+Feature: Searches with postcodes
+    Various searches involving postcodes
+
+    Scenario: US 5+4 ZIP codes are shortened to 5 ZIP codes if not found
+        When sending json search query "57701 1111, us" with address
+        Then result addresses contain
+            | postcode |
+            | 57701    |
+
+    Scenario: Postcode search with address
+        When sending json search query "9486, mauren"
+        Then at least 1 result is returned
+
+    Scenario: Postcode search with country
+        When sending json search query "9486, li" with address
+        Then result addresses contain
+            | country_code |
+            | li           |
+
+    Scenario: Postcode search with country code restriction
+        When sending json search query "9490" with address
+            | countrycodes |
+            | li |
+        Then result addresses contain
+            | country_code |
+            | li           |
+
+    Scenario: Postcode search with structured query
+        When sending json search query "" with address
+            | postalcode | country |
+            | 9490       | li |
+        Then result addresses contain
+            | country_code | postcode |
+            | li           | 9490     |

test/bdd/api/search/queries.feature

@@ -44,6 +44,17 @@ Feature: Search queries
           | country      | Deutschland |
           | country_code | de |
 
+    Scenario: With missing housenumber search falls back to road
+        When sending json search query "342 rocha, santa lucia" with address
+        Then address of result 0 is
+          | type         | value |
+          | road         | Rocha |
+          | city         | Santa Lucía |
+          | state        | Canelones |
+          | postcode     | 90700 |
+          | country      | Uruguay |
+          | country_code | uy |
+
     @Tiger
     Scenario: TIGER house number
         When sending json search query "323 22nd Street Southwest, Huron"
@@ -57,6 +68,77 @@ Feature: Search queries
           | place_rank |
           | 30 |
 
+    Scenario: Search with specific amenity
+        When sending json search query "[restaurant] Vaduz" with address
+        Then result addresses contain
+          | country |
+          | Liechtenstein |
+        And  results contain
+          | class   | type |
+          | amenity | restaurant |
+
+    Scenario: Search with key-value amenity
+        When sending json search query "[shop=hifi] hamburg"
+        Then results contain
+          | class | type |
+          | shop  | hifi |
+
+    Scenario: With multiple amenity search only the first is used
+        When sending json search query "[shop=hifi] [church] hamburg"
+        Then results contain
+          | class | type |
+          | shop  | hifi |
+
+    Scenario: With multiple amenity search only the first is used
+        When sending json search query "[church] [restaurant] hamburg"
+        Then results contain
+          | class   | type |
+          | amenity | place_of_worship |
+
+    Scenario: POI search near given coordinate
+        When sending json search query "restaurant near 47.16712,9.51100"
+        Then results contain
+          | class   | type |
+          | amenity | restaurant |
+
+    Scenario: Arbitrary key/value search near given coordinate
+        When sending json search query "[man_made=mast]  47.15739° N 9.61264° E"
+        Then results contain
+          | class    | type |
+          | man_made | mast |
+
+    Scenario: Arbitrary key/value search near given coordinate and named place
+        When sending json search query "[man_made=mast] amerlugalpe  47° 9′ 26″ N 9° 36′ 45″ E"
+        Then results contain
+          | class    | type |
+          | man_made | mast |
+
+    Scenario: Name search near given coordinate
+        When sending json search query "amerlugalpe, N 47.15739° E 9.61264°"
+        Then exactly 1 result is returned
+
+    Scenario: Name search near given coordinate without result
+        When sending json search query "amerlugalpe, N 47 15 7 W 9 61 26"
+        Then exactly 0 results are returned
+
+    Scenario: Arbitrary key/value search near a road
+        When sending json search query "[leisure=table_soccer_table] immenbusch"
+        Then results contain
+          | class   | type |
+          | leisure | table_soccer_table |
+
+    Scenario: Ignore other country codes in structured search with country
+        When sending json search query ""
+            | city | country |
+            | li   | de      |
+        Then exactly 0 results are returned
+
+    Scenario: Ignore country searches when query is restricted to countries
+        When sending json search query "de"
+            | countrycodes |
+            | li  |
+        Then exactly 0 results are returned
+
     # https://trac.openstreetmap.org/ticket/5094
     Scenario: housenumbers are ordered by complete match first
         When sending json search query "6395 geminis, montevideo" with address

test/bdd/api/search/simple.feature

@@ -102,12 +102,12 @@ Feature: Simple Tests
     Scenario: Empty XML search with viewbox
         When sending xml search query "xnznxvcx"
           | viewbox |
-          | 12,45.13,77,33 |
+          | 12,33,77,45.13 |
         Then result header contains
           | attr        | value |
           | querystring | xnznxvcx |
           | polygon     | false |
-          | viewbox     | 12,45.13,77,33 |
+          | viewbox     | 12,33,77,45.13 |
 
     Scenario: Empty XML search with viewboxlbrt
         When sending xml search query "xnznxvcx"
@@ -117,17 +117,17 @@ Feature: Simple Tests
           | attr        | value |
           | querystring | xnznxvcx |
           | polygon     | false |
-          | viewbox     | 12,45,77,34.13 |
+          | viewbox     | 12,34.13,77,45 |
 
     Scenario: Empty XML search with viewboxlbrt and viewbox
         When sending xml search query "pub"
           | viewbox        | viewboxblrt |
-          | 12,45.13,77,33 | 1,2,3,4 |
+          | 12,33,77,45.13 | 1,2,3,4 |
         Then result header contains
           | attr        | value |
           | querystring | pub |
           | polygon     | false |
-          | viewbox     | 12,45.13,77,33 |
+          | viewbox     | 12,33,77,45.13 |
 
     Scenario Outline: Empty XML search with polygon values
         When sending xml search query "xnznxvcx"

test/bdd/api/search/structured.feature

@@ -31,6 +31,33 @@ Feature: Structured search queries
           | attr        | value |
           | querystring | Old Palace Road, GU2 7UP, United Kingdom |
 
+    Scenario: Street with housenumber, city and postcode
+        When sending xml search query "" with address
+          | street             | city  | postalcode |
+          | 19 Am schrägen Weg | Vaduz | 9490       |
+        Then result addresses contain
+          | house_number | road |
+          | 19           | Am Schrägen Weg |
+
+    Scenario: Street with housenumber, city and bad postcode
+        When sending xml search query "" with address
+          | street             | city  | postalcode |
+          | 19 Am schrägen Weg | Vaduz | 9491       |
+        Then result addresses contain
+          | house_number | road |
+          | 19           | Am Schrägen Weg |
+
+    Scenario: Amenity, city
+        When sending json search query "" with address
+          | city  | amenity |
+          | Vaduz | church  |
+        Then result addresses contain
+          | country |
+          | Liechtenstein |
+        And  results contain
+          | class   | type |
+          | amenity | place_of_worship |
+
     Scenario: gihub #176
         When sending json search query "" with address
           | city |

test/bdd/db/import/interpolation.feature

@@ -248,10 +248,6 @@ Feature: Import of address interpolations
         Then results contain
          | ID | osm_type | osm_id |
          | 0  | W        | 11 |
-        When searching for "18 Cloud Street"
-        Then results contain
-         | ID | osm_type | osm_id |
-         | 0  | W        | 3 |
 
     Scenario: addr:street on housenumber way
         Given the scene parallel-road

test/bdd/db/import/linking.feature

@@ -40,11 +40,11 @@ Feature: Linking of places
     Scenario: Relations are not linked when in waterway relations
         Given the scene split-road
         And the places
-         | osm | class    | type  | name  | geometry |
+         | osm | class    | type   | name  | geometry |
-         | W1  | waterway | river | Rhein | :w-2 |
+         | W1  | waterway | stream | Rhein | :w-2 |
-         | W2  | waterway | river | Rhein | :w-3 |
+         | W2  | waterway | river  | Rhein | :w-3 |
-         | R1  | waterway | river | Rhein | :w-1 + :w-2 + :w-3 |
+         | R1  | waterway | river  | Rhein | :w-1 + :w-2 + :w-3 |
-         | R2  | waterway | river | Limmat| :w-4a |
+         | R2  | waterway | river  | Limmat| :w-4a |
         And the relations
          | id | members                          | tags+type |
          | 1  | R2                               | waterway  |
@@ -69,11 +69,11 @@ Feature: Linking of places
          | object | linked_place_id |
          | R1     | - |
 
-    Scenario: Waterways are not linked when waterway types don't match
+    Scenario: Waterways are not linked when the way type is not a river feature
         Given the scene split-road
         And the places
          | osm | class    | type     | name  | geometry |
-         | W1  | waterway | drain    | Rhein | :w-2 |
+         | W1  | waterway | lock     | Rhein | :w-2 |
          | R1  | waterway | river    | Rhein | :w-1 + :w-2 + :w-3 |
         And the relations
          | id | members               | tags+type |

test/bdd/db/import/placex.feature

@@ -79,13 +79,11 @@ Feature: Import into placex
          | osm  | class | type     | postcode | geometry |
          | N1   | place | postcode | EA452CD  | country:gb |
          | N2   | place | postcode | E45 23   | country:gb |
-         | N3   | place | postcode | y45      | country:gb |
         When importing
         Then placex contains
          | object | country_code | rank_search | rank_address |
          | N1     | gb           | 30          | 30 |
          | N2     | gb           | 30          | 30 |
-         | N3     | gb           | 30          | 30 |
 
     Scenario: search and address rank for DE postcodes correctly assigned
         Given the places

test/bdd/db/import/postcodes.feature

@@ -0,0 +1,115 @@
+@DB
+Feature: Import of postcodes
+    Tests for postcode estimation
+
+    Scenario: Postcodes on the object are prefered over those on the address
+        Given the scene admin-areas
+        And the named places
+            | osm | class    | type           | admin | addr+postcode | geometry |
+            | R1  | boundary | administrative | 6     | 112           | :b0      |
+            | R34 | boundary | administrative | 8     | 112 DE        | :b1:E    |
+            | R4  | boundary | administrative | 10    | 112 DE 34     | :b2:N    |
+        And the named places
+            | osm | class    | type        | addr+postcode | geometry |
+            | W93 | highway  | residential | 112 DE 344    | :w2N     |
+            | W22 | building | yes         | 112 DE 344N   | :building:w2N |
+        When importing
+        Then placex contains
+            | object | postcode    |
+            | W22    | 112 DE 344N |
+            | W93    | 112 DE 344  |
+            | R4     | 112 DE 34   |
+            | R34    | 112 DE      |
+            | R1     | 112         |
+
+    Scenario: Postcodes from a road are inherited by an attached building
+        Given the scene admin-areas
+        And the named places
+            | osm | class    | type        | addr+postcode | geometry |
+            | W93 | highway  | residential | 86034         | :w2N     |
+        And the named places
+            | osm | class    | type  | geometry |
+            | W22 | building | yes   | :building:w2N |
+        When importing
+        Then placex contains
+            | object | postcode | parent_place_id |
+            | W22    | 86034    | W93 |
+
+    Scenario: Postcodes from the lowest admin area are inherited by ways
+        Given the scene admin-areas
+        And the named places
+            | osm | class    | type           | admin | addr+postcode | geometry |
+            | R1  | boundary | administrative | 6     | 112           | :b0      |
+            | R34 | boundary | administrative | 8     | 112 DE        | :b1:E    |
+            | R4  | boundary | administrative | 10    | 112 DE 34     | :b2:N    |
+        And the named places
+            | osm | class    | type           | geometry |
+            | W93 | highway  | residential    | :w2N     |
+        When importing
+        Then placex contains
+            | object | postcode  |
+            | W93    | 112 DE 34 |
+
+    Scenario: Postcodes from the lowest admin area with postcode are inherited by ways
+        Given the scene admin-areas
+        And the named places
+            | osm | class    | type           | admin | addr+postcode | geometry |
+            | R1  | boundary | administrative | 6     | 112           | :b0      |
+            | R34 | boundary | administrative | 8     | 112 DE        | :b1:E    |
+        And the named places
+            | osm | class    | type           | admin | geometry |
+            | R4  | boundary | administrative | 10    | :b2:N    |
+        And the named places
+            | osm | class    | type           | geometry |
+            | W93 | highway  | residential    | :w2N     |
+        When importing
+        Then placex contains
+            | object | postcode | parent_place_id |
+            | W93    | 112 DE   | R4 |
+
+    Scenario: Postcodes from the lowest admin area are inherited by buildings
+        Given the scene admin-areas
+        And the named places
+            | osm | class    | type           | admin | addr+postcode | geometry |
+            | R1  | boundary | administrative | 6     | 112           | :b0      |
+            | R34 | boundary | administrative | 8     | 112 DE        | :b1:E    |
+            | R4  | boundary | administrative | 10    | 112 DE 34     | :b2:N    |
+        And the named places
+            | osm | class    | type  | geometry |
+            | W22 | building | yes   | :building:w2N |
+        When importing
+        Then placex contains
+            | object | postcode  |
+            | W22    | 112 DE 34 |
+
+    Scenario: Roads get postcodes from nearby buildings without other info
+        Given the scene admin-areas
+        And the named places
+            | osm | class    | type           | geometry |
+            | W93 | highway  | residential    | :w2N     |
+        And the named places
+            | osm | class    | type        | addr+postcode | geometry |
+            | W22 | building | yes         | 445023        | :building:w2N |
+        When importing
+        Then placex contains
+            | object | postcode |
+            | W93    | 445023   |
+
+    @wip
+    Scenario: Postcodes from admin boundaries are preferred over estimated postcodes
+        Given the scene admin-areas
+        And the named places
+            | osm | class    | type           | admin | addr+postcode | geometry |
+            | R1  | boundary | administrative | 6     | 112           | :b0      |
+            | R34 | boundary | administrative | 8     | 112 DE        | :b1:E    |
+            | R4  | boundary | administrative | 10    | 112 DE 34     | :b2:N    |
+        And the named places
+            | osm | class    | type           | geometry |
+            | W93 | highway  | residential    | :w2N     |
+        And the named places
+            | osm | class    | type        | addr+postcode | geometry |
+            | W22 | building | yes         | 445023        | :building:w2N |
+        When importing
+        Then placex contains
+            | object | postcode  |
+            | W93    | 112 DE 34 |

test/bdd/db/import/search_name.feature

@@ -23,17 +23,3 @@ Feature: Creation of search terms
         Then search_name contains
          | object | name_vector | nameaddress_vector |
          | N1     | foo         | the road |
-
-    Scenario: Roads take over the postcode from attached houses
-        Given the scene roads-with-pois
-        And the places
-         | osm | class | type  | housenr | postcode | street   | geometry |
-         | N1  | place | house | 1       | 12345    | North St | :p-S1 |
-        And the places
-         | osm | class   | type        | name     | geometry |
-         | W1  | highway | residential | North St | :w-north |
-        When importing
-        Then search_name contains
-         | object | nameaddress_vector |
-         | W1     | 12345 |
-

test/bdd/db/query/search_simple.feature

@@ -25,7 +25,9 @@ Feature: Searching of simple objects
           | osm | class    | type        | postcode | geometry |
           | R1  | boundary | postal_code | 54321    | poly-area:1.0 |
         And searching for "12345"
-        Then exactly 0 results are returned
+        Then results contain
+         | osm_type |
+         | P        |
         When searching for "54321"
         Then results contain
          | ID | osm_type | osm_id |

test/bdd/db/update/poi-inherited-postcode.feature

@@ -1,57 +0,0 @@
-@DB
-Feature: Update of POI-inherited poscode
-    Test updates of postcodes on street which was inherited from a related POI
-
-    Background: Street and house with postcode
-        Given the scene roads-with-pois
-        And the places
-         | osm | class | type  | housenr | postcode | street   | geometry |
-         | N1  | place | house | 1       | 12345    | North St |:p-S1 |
-        And the places
-         | osm | class   | type        | name     | geometry |
-         | W1  | highway | residential | North St | :w-north |
-        When importing
-        Then search_name contains
-         | object | nameaddress_vector |
-         | W1     | 12345 |
-
-    Scenario: POI-inherited postcode remains when way type is changed
-        When updating places
-         | osm | class   | type         | name     | geometry |
-         | W1  | highway | unclassified | North St | :w-north |
-        Then search_name contains
-         | object | nameaddress_vector |
-         | W1     | 12345 |
-
-    Scenario: POI-inherited postcode remains when way name is changed
-        When updating places
-         | osm | class   | type         | name     | geometry |
-         | W1  | highway | unclassified | South St | :w-north |
-        Then search_name contains
-         | object | nameaddress_vector |
-         | W1     | 12345 |
-
-    Scenario: POI-inherited postcode remains when way geometry is changed
-        When updating places
-         | osm | class   | type         | name     | geometry |
-         | W1  | highway | unclassified | South St | :w-south |
-        Then search_name contains
-         | object | nameaddress_vector |
-         | W1     | 12345 |
-
-    Scenario: POI-inherited postcode is added when POI postcode changes
-        When updating places
-         | osm | class | type  | housenr | postcode | street   | geometry |
-         | N1  | place | house | 1       | 54321    | North St |:p-S1 |
-        Then search_name contains
-         | object | nameaddress_vector |
-         | W1     | 54321 |
-
-    Scenario: POI-inherited postcode remains when POI geometry changes
-        When updating places
-         | osm | class | type  | housenr | postcode | street   | geometry |
-         | N1  | place | house | 1       | 12345    | North St |:p-S2 |
-        Then search_name contains
-         | object | nameaddress_vector |
-         | W1     | 12345 |
-

test/bdd/db/update/search_terms.feature

@@ -1,21 +0,0 @@
-@DB
-Feature: Update of search terms
-    Tests that search_name table is updated correctly
-
-    Scenario: POI-inherited postcode remains when another POI is deleted
-        Given the scene roads-with-pois
-        And the places
-         | osm | class | type  | housenr | postcode | street   | geometry |
-         | N1  | place | house | 1       | 12345    | North St |:p-S1 |
-         | N2  | place | house | 2       |          | North St |:p-S2 |
-        And the places
-         | osm | class   | type        | name     | geometry |
-         | W1  | highway | residential | North St | :w-north |
-        When importing
-        Then search_name contains
-         | object | nameaddress_vector |
-         | W1     | 12345 |
-        When marking for delete N2
-        Then search_name contains
-         | object | nameaddress_vector |
-         | W1     | 12345 |

test/bdd/steps/cgi-with-coverage.php

@@ -1,13 +1,20 @@
 <?php
 require_once 'SebastianBergmann/CodeCoverage/autoload.php';
+
+function coverage_shutdown($oCoverage)
+{
+    $oCoverage->stop();
+    $writer = new \SebastianBergmann\CodeCoverage\Report\PHP;
+    $writer->process($oCoverage, $_SERVER['PHP_CODE_COVERAGE_FILE']);
+}
+
 $covfilter = new SebastianBergmann\CodeCoverage\Filter();
 $covfilter->addDirectoryToWhitelist($_SERVER['COV_PHP_DIR']);
 $coverage = new SebastianBergmann\CodeCoverage\CodeCoverage(null, $covfilter);
 $coverage->start($_SERVER['COV_TEST_NAME']);
 
+register_shutdown_function('coverage_shutdown', $coverage);
+
 include $_SERVER['COV_SCRIPT_FILENAME'];
 
 
-$coverage->stop();
-$writer = new \SebastianBergmann\CodeCoverage\Report\PHP;
-$writer->process($coverage, $_SERVER['PHP_CODE_COVERAGE_FILE']);

test/bdd/steps/db_ops.py

@@ -287,7 +287,7 @@ def import_and_index_data_from_place_table(context):
               WHERE class='place' and type='houses' and osm_type='W'
                     and ST_GeometryType(geometry) = 'ST_LineString'""")
     context.db.commit()
-    context.nominatim.run_setup_script('index', 'index-noanalyse')
+    context.nominatim.run_setup_script('calculate-postcodes', 'index', 'index-noanalyse')
 
 @when("updating places")
 def update_place_table(context):

test/bdd/steps/queries.py

@@ -349,7 +349,7 @@ def website_search_request(context, fmt, query, addr):
 
     context.response = SearchResponse(outp, outfmt, status)
 
-@when(u'sending (?P<fmt>\S+ )?reverse coordinates (?P<lat>[0-9.-]+)?,(?P<lon>[0-9.-]+)?')
+@when(u'sending (?P<fmt>\S+ )?reverse coordinates (?P<lat>.+)?,(?P<lon>.+)?')
 def website_reverse_request(context, fmt, lat, lon):
     params = {}
     if lat is not None:

test/php/Nominatim/NearPointTest.php

@@ -1,90 +0,0 @@
-<?php
-
-namespace Nominatim;
-
-require '../../lib/NearPoint.php';
-
-class NearPointTest extends \PHPUnit_Framework_TestCase
-{
-
-
-    protected function setUp()
-    {
-    }
-
-    public function testExtractFromQuery()
-    {
-        // no coordinates expected
-        $this->assertFalse(NearPoint::extractFromQuery(''));
-        $this->assertFalse(NearPoint::extractFromQuery('abc'));
-        $this->assertFalse(NearPoint::extractFromQuery('12 34'));
-        $this->assertFalse(NearPoint::extractFromQuery('200.1 89.9')); // because latitude > 180
-
-        // coordinates expected
-        $this->assertNotNull(NearPoint::extractFromQuery('0.0 -0.0'));
-
-        $aRes = NearPoint::extractFromQuery(' abc 12.456 -78.90 def ');
-        $this->assertEquals($aRes['pt']->lat(), 12.456);
-        $this->assertEquals($aRes['pt']->lon(), -78.90);
-        $this->assertEquals($aRes['pt']->radius(), 0.1);
-        $this->assertEquals($aRes['query'], 'abc   def');
-
-        $aRes = NearPoint::extractFromQuery(' [12.456,-78.90] ');
-        $this->assertEquals($aRes['pt']->lat(), 12.456);
-        $this->assertEquals($aRes['pt']->lon(), -78.90);
-        $this->assertEquals($aRes['pt']->radius(), 0.1);
-        $this->assertEquals($aRes['query'], '');
-
-        // http://en.wikipedia.org/wiki/Geographic_coordinate_conversion
-        // these all represent the same location
-        $aQueries = array(
-                     '40 26.767 N 79 58.933 W',
-                     '40° 26.767′ N 79° 58.933′ W',
-                     "40° 26.767' N 79° 58.933' W",
-                     'N 40 26.767, W 79 58.933',
-                     'N 40°26.767′, W 79°58.933′',
-                     "N 40°26.767', W 79°58.933'",
- 
-                     '40 26 46 N 79 58 56 W',
-                     '40° 26′ 46″ N 79° 58′ 56″ W',
-                     'N 40 26 46 W 79 58 56',
-                     'N 40° 26′ 46″, W 79° 58′ 56″',
-                     'N 40° 26\' 46", W 79° 58\' 56"',
- 
-                     '40.446 -79.982',
-                     '40.446,-79.982',
-                     '40.446° N 79.982° W',
-                     'N 40.446° W 79.982°',
- 
-                     '[40.446 -79.982]',
-                     '       40.446  ,   -79.982     ',
-                    );
-
-
-        foreach ($aQueries as $sQuery) {
-            $aRes = NearPoint::extractFromQuery($sQuery);
-            $this->assertEquals(40.446, $aRes['pt']->lat(), 'degrees decimal ' . $sQuery, 0.01);
-            $this->assertEquals(-79.982, $aRes['pt']->lon(), 'degrees decimal ' . $sQuery, 0.01);
-        }
-    }
-
-    public function testWithinSQL()
-    {
-        $np = new NearPoint(0.1, 23, 1);
-
-        $this->assertEquals(
-            'ST_DWithin(foo, ST_SetSRID(ST_Point(23,0.1),4326), 1.000000)',
-            $np->withinSQL('foo')
-        );
-    }
-
-    public function testDistanceSQL()
-    {
-        $np = new NearPoint(0.1, 23, 1);
-
-        $this->assertEquals(
-            'ST_Distance(ST_SetSRID(ST_Point(23,0.1),4326), foo)',
-            $np->distanceSQL('foo')
-        );
-    }
-}

test/php/Nominatim/NominatimTest.php

@@ -2,7 +2,7 @@
 
 namespace Nominatim;
 
-require '../../lib/lib.php';
+require_once '../../lib/lib.php';
 
 class NominatimTest extends \PHPUnit_Framework_TestCase
 {
@@ -24,9 +24,9 @@ class NominatimTest extends \PHPUnit_Framework_TestCase
 
         $this->assertEquals(
             array(
-             'label' => "Country",
+             'label' => 'Country',
              'frequency' => 0,
-             'icon' => "poi_boundary_administrative",
+             'icon' => 'poi_boundary_administrative',
              'defzoom' => 6,
              'defdiameter' => 15,
              'importance' => 3
@@ -66,76 +66,6 @@ class NominatimTest extends \PHPUnit_Framework_TestCase
     }
 
 
-    public function testGetWordSets()
-    {
-        // given an array of arrays like
-        // array( array('a','b'), array('c','d') )
-        // returns a summary as string: '(a|b),(c|d)'
-
-
-        function serializeSets($aSets)
-        {
-            $aParts = array();
-            foreach ($aSets as $aSet) {
-                $aParts[] = '(' . join('|', $aSet) . ')';
-            }
-            return join(',', $aParts);
-        }
-
-        $this->assertEquals(
-            array(array('')),
-            getWordSets(array(), 0)
-        );
-
-        $this->assertEquals(
-            '(a)',
-            serializeSets(getWordSets(array("a"), 0))
-        );
-
-        $this->assertEquals(
-            '(a b),(a|b)',
-            serializeSets(getWordSets(array('a', 'b'), 0))
-        );
-
-        $this->assertEquals(
-            '(a b c),(a|b c),(a|b|c),(a b|c)',
-            serializeSets(getWordSets(array('a', 'b', 'c'), 0))
-        );
-
-        $this->assertEquals(
-            '(a b c d),(a|b c d),(a|b|c d),(a|b|c|d),(a|b c|d),(a b|c d),(a b|c|d),(a b c|d)',
-            serializeSets(getWordSets(array('a', 'b', 'c', 'd'), 0))
-        );
-
-
-        // Inverse
-        $this->assertEquals(
-            '(a b c),(c|a b),(c|b|a),(b c|a)',
-            serializeSets(getInverseWordSets(array('a', 'b', 'c'), 0))
-        );
-
-
-        // make sure we don't create too many sets
-        // 4 words => 8 sets
-        // 10 words => 511 sets
-        // 15 words => 12911 sets
-        // 18 words => 65536 sets
-        // 20 words => 169766 sets
-        // 22 words => 401930 sets
-        // 28 words => 3505699 sets (needs more than 4GB via 'phpunit -d memory_limit=' to run)
-        $this->assertEquals(
-            8,
-            count(getWordSets(array_fill(0, 4, 'a'), 0))
-        );
-
-
-        $this->assertEquals(
-            41226,
-            count(getWordSets(array_fill(0, 18, 'a'), 0))
-        );
-    }
-
-
     public function testCreatePointsAroundCenter()
     {
         // you might say we're creating a circle
@@ -203,4 +133,92 @@ class NominatimTest extends \PHPUnit_Framework_TestCase
             geometryText2Points('MULTIPOLYGON(((30 20, 45 40, 10 40, 30 20)),((15 5, 40 10, 10 20, 5 10, 15 5)))', $fRadius)
         );
     }
+
+    public function testParseLatLon()
+    {
+        // no coordinates expected
+        $this->assertFalse(parseLatLon(''));
+        $this->assertFalse(parseLatLon('abc'));
+        $this->assertFalse(parseLatLon('12 34'));
+
+        // coordinates expected
+        $this->assertNotNull(parseLatLon('0.0 -0.0'));
+
+        $aRes = parseLatLon(' abc 12.456 -78.90 def ');
+        $this->assertEquals($aRes[1], 12.456);
+        $this->assertEquals($aRes[2], -78.90);
+        $this->assertEquals($aRes[0], ' 12.456 -78.90 ');
+
+        $aRes = parseLatLon(' [12.456,-78.90] ');
+        $this->assertEquals($aRes[1], 12.456);
+        $this->assertEquals($aRes[2], -78.90);
+        $this->assertEquals($aRes[0], ' [12.456,-78.90] ');
+
+        $aRes = parseLatLon(' -12.456,-78.90 ');
+        $this->assertEquals($aRes[1], -12.456);
+        $this->assertEquals($aRes[2], -78.90);
+        $this->assertEquals($aRes[0], ' -12.456,-78.90 ');
+
+        // http://en.wikipedia.org/wiki/Geographic_coordinate_conversion
+        // these all represent the same location
+        $aQueries = array(
+                     '40 26.767 N 79 58.933 W',
+                     '40° 26.767′ N 79° 58.933′ W',
+                     "40° 26.767' N 79° 58.933' W",
+                     'N 40 26.767, W 79 58.933',
+                     'N 40°26.767′, W 79°58.933′',
+                     "N 40°26.767', W 79°58.933'",
+ 
+                     '40 26 46 N 79 58 56 W',
+                     '40° 26′ 46″ N 79° 58′ 56″ W',
+                     'N 40 26 46 W 79 58 56',
+                     'N 40° 26′ 46″, W 79° 58′ 56″',
+                     'N 40° 26\' 46", W 79° 58\' 56"',
+ 
+                     '40.446 -79.982',
+                     '40.446,-79.982',
+                     '40.446° N 79.982° W',
+                     'N 40.446° W 79.982°',
+ 
+                     '[40.446 -79.982]',
+                     '       40.446  ,   -79.982     ',
+                    );
+
+
+        foreach ($aQueries as $sQuery) {
+            $aRes = parseLatLon($sQuery);
+            $this->assertEquals(40.446, $aRes[1], 'degrees decimal ' . $sQuery, 0.01);
+            $this->assertEquals(-79.982, $aRes[2], 'degrees decimal ' . $sQuery, 0.01);
+            $this->assertEquals($sQuery, $aRes[0]);
+        }
+    }
+
+    private function closestHouseNumberEvenOddOther($startnumber, $endnumber, $fraction, $aExpected)
+    {
+        foreach (['even', 'odd', 'other'] as $itype) {
+            $this->assertEquals(
+                $aExpected[$itype],
+                closestHouseNumber([
+                                    'startnumber' => $startnumber,
+                                    'endnumber' => $endnumber,
+                                    'fraction' => $fraction,
+                                    'interpolationtype' => $itype
+                                   ]),
+                "$startnumber => $endnumber, $fraction, $itype"
+            );
+        }
+    }
+
+    public function testClosestHouseNumber()
+    {
+        $this->closestHouseNumberEvenOddOther(50, 100, 0.5, ['even' => 76, 'odd' => 75, 'other' => 75]);
+        // upper bound
+        $this->closestHouseNumberEvenOddOther(50, 100, 1.5, ['even' => 100, 'odd' => 100, 'other' => 100]);
+        // lower bound
+        $this->closestHouseNumberEvenOddOther(50, 100, -0.5, ['even' => 50, 'odd' => 50, 'other' => 50]);
+        // fraction 0
+        $this->closestHouseNumberEvenOddOther(50, 100, 0, ['even' => 50, 'odd' => 51, 'other' => 50]);
+        // start == end
+        $this->closestHouseNumberEvenOddOther(50, 50, 0.5, ['even' => 50, 'odd' => 50, 'other' => 50]);
+    }
 }

test/php/Nominatim/PhraseTest.php

@@ -0,0 +1,87 @@
+<?php
+
+namespace Nominatim;
+
+require_once '../../lib/Phrase.php';
+
+class PhraseTest extends \PHPUnit_Framework_TestCase
+{
+    private function serializeSets($aSets)
+    {
+        $aParts = array();
+        foreach ($aSets as $aSet) {
+            $aParts[] = '(' . join('|', $aSet) . ')';
+        }
+        return join(',', $aParts);
+    }
+
+
+    public function testEmptyPhrase()
+    {
+        $oPhrase = new Phrase('', '');
+
+        $this->assertEquals(
+            array(array('')),
+            $oPhrase->getWordSets()
+        );
+    }
+
+
+    public function testSingleWordPhrase()
+    {
+        $oPhrase = new Phrase('a', '');
+
+        $this->assertEquals(
+            '(a)',
+            $this->serializeSets($oPhrase->getWordSets())
+        );
+    }
+
+
+    public function testMultiWordPhrase()
+    {
+        $oPhrase = new Phrase('a b', '');
+        $this->assertEquals(
+            '(a b),(a|b)',
+            $this->serializeSets($oPhrase->getWordSets())
+        );
+
+        $oPhrase = new Phrase('a b c', '');
+        $this->assertEquals(
+            '(a b c),(a|b c),(a|b|c),(a b|c)',
+            $this->serializeSets($oPhrase->getWordSets())
+        );
+
+        $oPhrase = new Phrase('a b c d', '');
+        $this->assertEquals(
+            '(a b c d),(a|b c d),(a|b|c d),(a|b|c|d),(a|b c|d),(a b|c d),(a b|c|d),(a b c|d)',
+            $this->serializeSets($oPhrase->getWordSets())
+        );
+    }
+
+
+    public function testInverseWordSets()
+    {
+        $oPhrase = new Phrase('a b c', '');
+        $oPhrase->invertWordSets();
+
+        $this->assertEquals(
+            '(a b c),(c|a b),(c|b|a),(b c|a)',
+            $this->serializeSets($oPhrase->getWordSets())
+        );
+    }
+
+
+    public function testMaxDepth()
+    {
+        $oPhrase = new Phrase(join(' ', array_fill(0, 4, 'a')), '');
+        $this->assertEquals(8, count($oPhrase->getWordSets()));
+        $oPhrase->invertWordSets();
+        $this->assertEquals(8, count($oPhrase->getWordSets()));
+
+        $oPhrase = new Phrase(join(' ', array_fill(0, 18, 'a')), '');
+        $this->assertEquals(41226, count($oPhrase->getWordSets()));
+        $oPhrase->invertWordSets();
+        $this->assertEquals(41226, count($oPhrase->getWordSets()));
+    }
+}

test/php/Nominatim/SearchContextTest.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace Nominatim;
+
+@define('CONST_BasePath', '../../');
+
+require_once '../../lib/SearchContext.php';
+
+class SearchContextTest extends \PHPUnit_Framework_TestCase
+{
+    private $oCtx;
+
+    protected function setUp()
+    {
+        $this->oCtx = new SearchContext();
+    }
+
+    public function testHasNearPoint()
+    {
+        $this->assertFalse($this->oCtx->hasNearPoint());
+        $this->oCtx->setNearPoint(0, 0);
+        $this->assertTrue($this->oCtx->hasNearPoint());
+    }
+
+    public function testNearRadius()
+    {
+        $this->oCtx->setNearPoint(1, 1);
+        $this->assertEquals(0.1, $this->oCtx->nearRadius());
+        $this->oCtx->setNearPoint(1, 1, 0.338);
+        $this->assertEquals(0.338, $this->oCtx->nearRadius());
+    }
+
+    public function testWithinSQL()
+    {
+        $this->oCtx->setNearPoint(0.1, 23, 1);
+
+        $this->assertEquals(
+            'ST_DWithin(foo, ST_SetSRID(ST_Point(23,0.1),4326), 1.000000)',
+            $this->oCtx->withinSQL('foo')
+        );
+    }
+
+    public function testDistanceSQL()
+    {
+        $this->oCtx->setNearPoint(0.1, 23, 1);
+
+        $this->assertEquals(
+            'ST_Distance(ST_SetSRID(ST_Point(23,0.1),4326), foo)',
+            $this->oCtx->distanceSQL('foo')
+        );
+    }
+}

test/scenes/bin/osm2wkt.cc

@@ -7,19 +7,49 @@
 #include <unordered_map>
 
 #include <osmium/area/assembler.hpp>
-#include <osmium/area/multipolygon_collector.hpp>
+#include <osmium/area/multipolygon_manager.hpp>
-#include <osmium/area/problem_reporter_exception.hpp>
+
 #include <osmium/geom/wkt.hpp>
 #include <osmium/handler.hpp>
 #include <osmium/handler/node_locations_for_ways.hpp>
 #include <osmium/io/any_input.hpp>
 #include <osmium/visitor.hpp>
+#include <osmium/object_pointer_collection.hpp>
 #include <osmium/index/map/sparse_mem_array.hpp>
+#include <osmium/osm/object_comparisons.hpp>
 
 typedef osmium::index::map::SparseMemArray<osmium::unsigned_object_id_type, osmium::Location> index_type;
 
 typedef osmium::handler::NodeLocationsForWays<index_type, index_type> location_handler_type;
 
+struct AbsoluteIdHandler : public osmium::handler::Handler {
+
+    enum { BASE = 100000000 };
+
+    void node(osmium::Node& o) {
+        if (o.id() < 0)
+            o.set_id(BASE-o.id());
+    }
+
+    void way(osmium::Way& o) {
+        if (o.id() < 0)
+            o.set_id(BASE-o.id());
+
+        for (osmium::NodeRef &n: o.nodes())
+            if (n.ref() < 0)
+                n.set_ref(BASE-n.ref());
+    }
+
+    void relation(osmium::Relation& o) {
+        if (o.id() < 0)
+            o.set_id(BASE-o.id());
+
+        for (auto &m : o.members())
+            if (m.ref() < 0)
+                m.set_ref(BASE-m.ref());
+    }
+};
+
 
 class ExportToWKTHandler : public osmium::handler::Handler {
 
@@ -33,7 +63,8 @@ public:
     }
 
     void way(const osmium::Way& way) {
-        if (!way.is_closed() || !way.tags().get_value_by_key("area"))
+        if (!way.nodes().empty()
+            && (!way.is_closed() || !way.tags().get_value_by_key("area")))
             print_geometry(way.tags(), m_factory.create_linestring(way));
     }
 
@@ -48,7 +79,6 @@ public:
     }
 
 private:
-
     void print_geometry(const osmium::TagList& tags, const std::string& wkt) {
         const char* scenario = tags.get_value_by_key("test:section");
         const char* id = tags.get_value_by_key("test:id");
@@ -68,28 +98,42 @@ int main(int argc, char* argv[]) {
         exit(1);
     }
 
-    std::string input_filename {argv[1]};
+    osmium::io::File input_file{argv[1]};
 
-    osmium::area::ProblemReporterException problem_reporter;
+    // need to sort the data first and make ids absolute
-    osmium::area::Assembler::config_type assembler_config(&problem_reporter);
+    std::cerr << "Read file...\n";
-    osmium::area::MultipolygonCollector<osmium::area::Assembler> collector(assembler_config);
+    osmium::io::Reader reader{input_file};
+    std::vector<osmium::memory::Buffer> changes;
+    osmium::ObjectPointerCollection objects;
+    AbsoluteIdHandler abshandler;
+    while (osmium::memory::Buffer buffer = reader.read()) {
+            osmium::apply(buffer, abshandler, objects);
+            changes.push_back(std::move(buffer));
+    }
+    reader.close();
+
+    std::cerr << "Sort file...\n";
+    objects.sort(osmium::object_order_type_id_version());
+
+    osmium::area::Assembler::config_type assembler_config;
+    osmium::area::MultipolygonManager<osmium::area::Assembler> mp_manager{assembler_config};
 
     std::cerr << "Pass 1...\n";
-    osmium::io::Reader reader1(input_filename, osmium::osm_entity_bits::relation);
-    collector.read_relations(reader1);
-    std::cerr << "Pass 1 done\n";
-
     index_type index_pos;
     index_type index_neg;
     location_handler_type location_handler(index_pos, index_neg);
+    ExportToWKTHandler export_handler;
+    osmium::apply(objects.begin(), objects.end(), location_handler,
+                  export_handler, mp_manager);
+    mp_manager.prepare_for_lookup();
+    std::cerr << "Pass 1 done\n";
+
 
     std::cerr << "Pass 2...\n";
-    ExportToWKTHandler export_handler;
+    osmium::apply(objects.cbegin(), objects.cend(), mp_manager.handler([&export_handler](osmium::memory::Buffer&& buffer) {
-    osmium::io::Reader reader2(input_filename);
-    osmium::apply(reader2, location_handler, export_handler, collector.handler([&export_handler](osmium::memory::Buffer&& buffer) {
         osmium::apply(buffer, export_handler);
     }));
-    reader2.close();
+
     export_handler.close();
     std::cerr << "Pass 2 done\n";
 }

test/scenes/data/admin-areas.wkt

@@ -0,0 +1,19 @@
+c1:N |  POINT(73.8419358 60.0763887)
+c1:E |  POINT(73.8393798 60.0488584)
+c0 |  POINT(73.8679209 60.0588527)
+c2:N |  POINT(73.896249 60.0631047)
+c2:S |  POINT(73.8932671 60.0434346)
+c2:E |  POINT(73.9162704 60.0471569)
+c1:W |  POINT(73.8990179 60.055876)
+c2:W |  POINT(73.8568453 60.0597032)
+w2N |  LINESTRING(73.8836825 60.0612977,73.8880489 60.0598094,73.8953972 60.0601283,73.9033844 60.058959)
+w1W:2W |  LINESTRING(73.8523722 60.0497092,73.85791 60.0520485,73.8617439 60.0573645,73.8706896 60.0554508)
+building:w2N |  LINESTRING(73.8963618 60.0604955,73.8961463 60.0602249,73.8967091 60.0601132,73.8969246 60.0603838,73.8963618 60.0604955)
+b0 |  MULTIPOLYGON(((73.8012539 60.0573645,73.8225532 60.0371591,73.8493903 60.035457,73.8843212 60.0356698,73.9049815 60.0358825,73.9192521 60.0356698,73.9260679 60.0514105,73.9216633 60.0591056,73.9141402 60.0722448,73.8804873 60.070332,73.8719676 60.0917916,73.8255351 60.0875433,73.8084956 60.0758576,73.8012539 60.0573645)))
+b1:N |  MULTIPOLYGON(((73.8012539 60.0573645,73.8447045 60.0611915,73.8692843 60.0674706,73.8804873 60.070332,73.8719676 60.0917916,73.8255351 60.0875433,73.8084956 60.0758576,73.8012539 60.0573645)))
+b2:S |  MULTIPOLYGON(((73.8694117 60.0507725,73.8843212 60.0356698,73.9049815 60.0358825,73.9075368 60.0523758,73.8830432 60.0517295,73.8694117 60.0507725)))
+b1:W |  MULTIPOLYGON(((73.8012539 60.0573645,73.8225532 60.0371591,73.8493903 60.035457,73.8843212 60.0356698,73.8694117 60.0507725,73.8447045 60.0611915,73.8012539 60.0573645)))
+b1:E |  MULTIPOLYGON(((73.8447045 60.0611915,73.8694117 60.0507725,73.8843212 60.0356698,73.9049815 60.0358825,73.9192521 60.0356698,73.9260679 60.0514105,73.9216633 60.0591056,73.9141402 60.0722448,73.8804873 60.070332,73.8692843 60.0674706,73.8447045 60.0611915)))
+b2:E |  MULTIPOLYGON(((73.9049815 60.0358825,73.9192521 60.0356698,73.9260679 60.0514105,73.9216633 60.0591056,73.9075368 60.0523758,73.9049815 60.0358825)))
+b2:N |  MULTIPOLYGON(((73.8692843 60.0674706,73.8830432 60.0517295,73.9075368 60.0523758,73.9216633 60.0591056,73.9141402 60.0722448,73.8804873 60.070332,73.8692843 60.0674706)))
+b2:W |  MULTIPOLYGON(((73.8447045 60.0611915,73.8694117 60.0507725,73.8830432 60.0517295,73.8692843 60.0674706,73.8447045 60.0611915)))

test/scenes/data/admin.osm

@@ -0,0 +1,250 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<osm version='0.6' upload='false' generator='JOSM'>
+  <node id='-30473' action='modify' lat='60.07585759191' lon='73.80849562007' />
+  <node id='-30475' action='modify' lat='60.05736451143' lon='73.80125385169' />
+  <node id='-30477' action='modify' lat='60.0371590755' lon='73.82255317047' />
+  <node id='-30479' action='modify' lat='60.03545700058' lon='73.84939031213' />
+  <node id='-30481' action='modify' lat='60.03566976474' lon='73.88432119493' />
+  <node id='-30483' action='modify' lat='60.03566976474' lon='73.91925207773' />
+  <node id='-30485' action='modify' lat='60.05141051018' lon='73.92606785974' />
+  <node id='-30487' action='modify' lat='60.07224481634' lon='73.91414024122' />
+  <node id='-30489' action='modify' lat='60.07033201023' lon='73.88048731755' />
+  <node id='-30491' action='modify' lat='60.09179158393' lon='73.87196759004' />
+  <node id='-30493' action='modify' lat='60.08754327238' lon='73.8255350751' />
+  <node id='-30495' action='modify' lat='60.06119151655' lon='73.844704462' />
+  <node id='-30497' action='modify' lat='60.05077251777' lon='73.86941167178' />
+  <node id='-30499' action='modify' lat='60.05172950176' lon='73.8830432358' />
+  <node id='-30501' action='modify' lat='60.06747055357' lon='73.86928433032' />
+  <node id='-30503' action='modify' lat='60.05910557298' lon='73.92166332136' />
+  <node id='-30505' action='modify' lat='60.05237575233' lon='73.90753676249' />
+  <node id='-30507' action='modify' lat='60.03588252753' lon='73.90498153415' />
+  <node id='-30509' action='modify' lat='60.07638874281' lon='73.84193576355'>
+    <tag k='test:id' v='c1:N' />
+    <tag k='test:section' v='admin-areas' />
+  </node>
+  <node id='-30511' action='modify' lat='60.04885836023' lon='73.8393798453'>
+    <tag k='test:id' v='c1:E' />
+    <tag k='test:section' v='admin-areas' />
+  </node>
+  <node id='-30513' action='modify' lat='60.05885273763' lon='73.86792093246'>
+    <tag k='test:id' v='c0' />
+    <tag k='test:section' v='admin-areas' />
+  </node>
+  <node id='-30515' action='modify' lat='60.06310474639' lon='73.89624902644'>
+    <tag k='test:id' v='c2:N' />
+    <tag k='test:section' v='admin-areas' />
+  </node>
+  <node id='-30517' action='modify' lat='60.04343461246' lon='73.89326712181'>
+    <tag k='test:id' v='c2:S' />
+    <tag k='test:section' v='admin-areas' />
+  </node>
+  <node id='-30519' action='modify' lat='60.04715688821' lon='73.91627038609'>
+    <tag k='test:id' v='c2:E' />
+    <tag k='test:section' v='admin-areas' />
+  </node>
+  <node id='-30521' action='modify' lat='60.05587600549' lon='73.89901793788'>
+    <tag k='test:id' v='c1:W' />
+    <tag k='test:section' v='admin-areas' />
+  </node>
+  <node id='-30523' action='modify' lat='60.05970318321' lon='73.8568452867'>
+    <tag k='test:id' v='c2:W' />
+    <tag k='test:section' v='admin-areas' />
+  </node>
+  <node id='-30525' action='modify' lat='60.06129765646' lon='73.88368253486' />
+  <node id='-30527' action='modify' lat='60.05980943422' lon='73.88804889521' />
+  <node id='-30529' action='modify' lat='60.06012834464' lon='73.89539716019' />
+  <node id='-30531' action='modify' lat='60.05895899137' lon='73.90338440473' />
+  <node id='-30533' action='modify' lat='60.04970916969' lon='73.85237221676' />
+  <node id='-30535' action='modify' lat='60.05204849025' lon='73.85791003964' />
+  <node id='-30537' action='modify' lat='60.05736451143' lon='73.86174391702' />
+  <node id='-30539' action='modify' lat='60.05545084244' lon='73.87068963091' />
+  <node id='-30541' action='modify' lat='60.06049547301' lon='73.89636177639' />
+  <node id='-30543' action='modify' lat='60.06022493568' lon='73.89614625694' />
+  <node id='-30545' action='modify' lat='60.06011324975' lon='73.89670909505' />
+  <node id='-30547' action='modify' lat='60.060383788' lon='73.89692461449' />
+  <node id='100000' action='delete' timestamp='2014-01-01T00:00:00Z' uid='1' user='test' version='1' changeset='1' lat='3.0' lon='2.0' />
+  <node id='100001' action='delete' timestamp='2014-01-01T00:00:00Z' uid='1' user='test' version='1' changeset='1' lat='3.5' lon='2.0' />
+  <node id='100002' action='delete' timestamp='2014-01-01T00:00:00Z' uid='1' user='test' version='1' changeset='1' lat='3.5' lon='2.5' />
+  <node id='100003' action='delete' timestamp='2014-01-01T00:00:00Z' uid='1' user='test' version='1' changeset='1' lat='3.0' lon='2.5' />
+  <way id='-30553' action='modify'>
+    <nd ref='-30489' />
+    <nd ref='-30491' />
+    <nd ref='-30493' />
+    <nd ref='-30473' />
+    <nd ref='-30475' />
+  </way>
+  <way id='-30555' action='modify'>
+    <nd ref='-30495' />
+    <nd ref='-30501' />
+  </way>
+  <way id='-30557' action='modify'>
+    <nd ref='-30495' />
+    <nd ref='-30497' />
+  </way>
+  <way id='-30559' action='modify'>
+    <nd ref='-30497' />
+    <nd ref='-30499' />
+  </way>
+  <way id='-30561' action='modify'>
+    <nd ref='-30499' />
+    <nd ref='-30505' />
+  </way>
+  <way id='-30563' action='modify'>
+    <nd ref='-30505' />
+    <nd ref='-30507' />
+  </way>
+  <way id='-30565' action='modify'>
+    <nd ref='-30503' />
+    <nd ref='-30487' />
+    <nd ref='-30489' />
+  </way>
+  <way id='-30567' action='modify'>
+    <nd ref='-30507' />
+    <nd ref='-30483' />
+    <nd ref='-30485' />
+    <nd ref='-30503' />
+  </way>
+  <way id='-30569' action='modify'>
+    <nd ref='-30481' />
+    <nd ref='-30507' />
+  </way>
+  <way id='-30571' action='modify'>
+    <nd ref='-30475' />
+    <nd ref='-30477' />
+    <nd ref='-30479' />
+    <nd ref='-30481' />
+  </way>
+  <way id='-30573' action='modify'>
+    <nd ref='-30475' />
+    <nd ref='-30495' />
+  </way>
+  <way id='-30575' action='modify'>
+    <nd ref='-30501' />
+    <nd ref='-30489' />
+  </way>
+  <way id='-30577' action='modify'>
+    <nd ref='-30497' />
+    <nd ref='-30481' />
+  </way>
+  <way id='-30579' action='modify'>
+    <nd ref='-30505' />
+    <nd ref='-30503' />
+  </way>
+  <way id='-30581' action='modify'>
+    <nd ref='-30499' />
+    <nd ref='-30501' />
+  </way>
+  <way id='-30583' action='modify'>
+    <nd ref='-30525' />
+    <nd ref='-30527' />
+    <nd ref='-30529' />
+    <nd ref='-30531' />
+    <tag k='test:id' v='w2N' />
+    <tag k='test:section' v='admin-areas' />
+  </way>
+  <way id='-30585' action='modify'>
+    <nd ref='-30533' />
+    <nd ref='-30535' />
+    <nd ref='-30537' />
+    <nd ref='-30539' />
+    <tag k='test:id' v='w1W:2W' />
+    <tag k='test:section' v='admin-areas' />
+  </way>
+  <way id='-30587' action='modify'>
+    <nd ref='-30541' />
+    <nd ref='-30543' />
+    <nd ref='-30545' />
+    <nd ref='-30547' />
+    <nd ref='-30541' />
+    <tag k='test:id' v='building:w2N' />
+    <tag k='test:section' v='admin-areas' />
+  </way>
+  <way id='100000' action='delete' timestamp='2014-01-01T00:00:00Z' uid='1' user='test' version='1' changeset='1'>
+    <tag k='note' v='test area, do not leave' />
+  </way>
+  <relation id='-30590' action='modify'>
+    <member type='way' ref='-30553' role='' />
+    <member type='way' ref='-30571' role='' />
+    <member type='way' ref='-30569' role='' />
+    <member type='way' ref='-30567' role='' />
+    <member type='way' ref='-30565' role='' />
+    <tag k='boundary' v='administrative' />
+    <tag k='test:id' v='b0' />
+    <tag k='test:section' v='admin-areas' />
+    <tag k='type' v='multipolygon' />
+  </relation>
+  <relation id='-30592' action='modify'>
+    <member type='way' ref='-30553' role='' />
+    <member type='way' ref='-30573' role='' />
+    <member type='way' ref='-30555' role='' />
+    <member type='way' ref='-30575' role='' />
+    <tag k='boundary' v='administrative' />
+    <tag k='test:id' v='b1:N' />
+    <tag k='test:section' v='admin-areas' />
+    <tag k='type' v='multipolygon' />
+  </relation>
+  <relation id='-30594' action='modify'>
+    <member type='way' ref='-30571' role='' />
+    <member type='way' ref='-30573' role='' />
+    <member type='way' ref='-30557' role='' />
+    <member type='way' ref='-30577' role='' />
+    <tag k='boundary' v='administrative' />
+    <tag k='test:id' v='b1:W' />
+    <tag k='test:section' v='admin-areas' />
+    <tag k='type' v='multipolygon' />
+  </relation>
+  <relation id='-30596' action='modify'>
+    <member type='way' ref='-30565' role='' />
+    <member type='way' ref='-30567' role='' />
+    <member type='way' ref='-30569' role='' />
+    <member type='way' ref='-30577' role='' />
+    <member type='way' ref='-30557' role='' />
+    <member type='way' ref='-30555' role='' />
+    <member type='way' ref='-30575' role='' />
+    <tag k='boundary' v='administrative' />
+    <tag k='test:id' v='b1:E' />
+    <tag k='test:section' v='admin-areas' />
+    <tag k='type' v='multipolygon' />
+  </relation>
+  <relation id='-30598' action='modify'>
+    <member type='way' ref='-30565' role='' />
+    <member type='way' ref='-30579' role='' />
+    <member type='way' ref='-30561' role='' />
+    <member type='way' ref='-30581' role='' />
+    <member type='way' ref='-30575' role='' />
+    <tag k='boundary' v='administrative' />
+    <tag k='test:id' v='b2:N' />
+    <tag k='test:section' v='admin-areas' />
+    <tag k='type' v='multipolygon' />
+  </relation>
+  <relation id='-30600' action='modify'>
+    <member type='way' ref='-30555' role='' />
+    <member type='way' ref='-30557' role='' />
+    <member type='way' ref='-30559' role='' />
+    <member type='way' ref='-30581' role='' />
+    <tag k='boundary' v='administrative' />
+    <tag k='test:id' v='b2:W' />
+    <tag k='test:section' v='admin-areas' />
+    <tag k='type' v='multipolygon' />
+  </relation>
+  <relation id='-30602' action='modify'>
+    <member type='way' ref='-30577' role='' />
+    <member type='way' ref='-30559' role='' />
+    <member type='way' ref='-30561' role='' />
+    <member type='way' ref='-30563' role='' />
+    <member type='way' ref='-30569' role='' />
+    <tag k='boundary' v='administrative' />
+    <tag k='test:id' v='b2:S' />
+    <tag k='test:section' v='admin-areas' />
+    <tag k='type' v='multipolygon' />
+  </relation>
+  <relation id='-30604' action='modify'>
+    <member type='way' ref='-30579' role='' />
+    <member type='way' ref='-30567' role='' />
+    <member type='way' ref='-30563' role='' />
+    <tag k='boundary' v='administrative' />
+    <tag k='test:id' v='b2:E' />
+    <tag k='test:section' v='admin-areas' />
+    <tag k='type' v='multipolygon' />
+  </relation>
+</osm>

test/scenes/data/building-on-street-corner.wkt

@@ -4,4 +4,4 @@ n-outer |  POINT(1.0039478 2.0004676)
 n-edge-WE |  POINT(1.0039599 2.0002345)
 w-WE |  LINESTRING(1.0031759 2.0002316,1.0040361 2.0002211,1.0042735 2.0002264)
 w-NS |  LINESTRING(1.0040414 2.0001051,1.0040361 2.0002211,1.0040364 2.0006377)
-w-building |  MULTIPOLYGON(((1.0040019 2.000324,1.0040016 2.0002344,1.0039599 2.0002345,1.0039037 2.0002347,1.0039043 2.0004389,1.0040023 2.0004386,1.0040019 2.000324)))
+w-building |  MULTIPOLYGON(((1.0039037 2.0002347,1.0039599 2.0002345,1.0040016 2.0002344,1.0040019 2.000324,1.0040023 2.0004386,1.0039043 2.0004389,1.0039037 2.0002347)))

test/scenes/data/poly-area.wkt

@@ -1,11 +1,11 @@
-0.0001 |  MULTIPOLYGON(((0.001 0,0 0,0 0.1,0.001 0.1,0.001 0)))
+0.0001 |  MULTIPOLYGON(((0 0,0.001 0,0.001 0.1,0 0.1,0 0)))
-0.0005 |  MULTIPOLYGON(((0.005 0,0 0,0 0.1,0.005 0.1,0.005 0)))
+0.0005 |  MULTIPOLYGON(((0 0,0.005 0,0.005 0.1,0 0.1,0 0)))
-0.001 |  MULTIPOLYGON(((0.01 0,0 0,0 0.1,0.01 0.1,0.01 0)))
+0.001 |  MULTIPOLYGON(((0 0,0.01 0,0.01 0.1,0 0.1,0 0)))
-0.005 |  MULTIPOLYGON(((0.05 0,0 0,0 0.1,0.05 0.1,0.05 0)))
+0.005 |  MULTIPOLYGON(((0 0,0.05 0,0.05 0.1,0 0.1,0 0)))
-0.01 |  MULTIPOLYGON(((0.1 0,0 0,0 0.1,0.1 0.1,0.1 0)))
+0.01 |  MULTIPOLYGON(((0 0,0.1 0,0.1 0.1,0 0.1,0 0)))
-0.05 |  MULTIPOLYGON(((0.5 0,0 0,0 0.1,0.5 0.1,0.5 0)))
+0.05 |  MULTIPOLYGON(((0 0,0.5 0,0.5 0.1,0 0.1,0 0)))
-0.1 |  MULTIPOLYGON(((0.1 0,0 0,0 1,0.1 1,0.1 0)))
+0.1 |  MULTIPOLYGON(((0 0,0.1 0,0.1 1,0 1,0 0)))
-0.5 |  MULTIPOLYGON(((0.5 0,0 0,0 1,0.5 1,0.5 0)))
+0.5 |  MULTIPOLYGON(((0 0,0.5 0,0.5 1,0 1,0 0)))
-1.0 |  MULTIPOLYGON(((1 0,0 0,0 1,1 1,1 0)))
+1.0 |  MULTIPOLYGON(((0 0,1 0,1 1,0 1,0 0)))
-2.0 |  MULTIPOLYGON(((2 0,0 0,0 1,2 1,2 0)))
+2.0 |  MULTIPOLYGON(((0 0,2 0,2 1,0 1,0 0)))
-5.0 |  MULTIPOLYGON(((5 0,0 0,0 1,5 1,5 0)))
+5.0 |  MULTIPOLYGON(((0 0,5 0,5 1,0 1,0 0)))

test/scenes/data/way-area-with-center.wkt

@@ -1,5 +1,5 @@
-inner-C |  POINT(0.0035625 -0.0066188)
-outer-C |  POINT(0.0041244 -0.0060007)
-inner-N |  POINT(0.0018846 -0.0023652)
 inner-S |  POINT(0.0048516 -0.0095176)
-area |  MULTIPOLYGON(((0.0077125 -0.0066566,0.0065469 -0.0099414,0.0038979 -0.0109481,0.0026794 -0.0105772,0.0022025 -0.0099944,0.0026264 -0.0091997,0.0026264 -0.0080341,0.0019376 -0.0065507,0.0010369 -0.0072924,0.0005071 -0.0060738,0.0017787 -0.00565,0.0005071 -0.0042195,0.0005601 -0.0025771,0.0013019 -0.0015175,0.0050105 -0.0021533,0.006441 -0.0025771,0.0075006 -0.0040076,0.0033681 -0.0059149,0.0051694 -0.0076633,0.0061231 -0.0064977,0.0068648 -0.0049612,0.0077125 -0.0066566)))
+inner-N |  POINT(0.0018846 -0.0023652)
+outer-C |  POINT(0.0041244 -0.0060007)
+inner-C |  POINT(0.0035625 -0.0066188)
+area |  MULTIPOLYGON(((0.0005071 -0.0060738,0.0010369 -0.0072924,0.0019376 -0.0065507,0.0026264 -0.0080341,0.0026264 -0.0091997,0.0022025 -0.0099944,0.0026794 -0.0105772,0.0038979 -0.0109481,0.0065469 -0.0099414,0.0077125 -0.0066566,0.0068648 -0.0049612,0.0061231 -0.0064977,0.0051694 -0.0076633,0.0033681 -0.0059149,0.0075006 -0.0040076,0.006441 -0.0025771,0.0050105 -0.0021533,0.0013019 -0.0015175,0.0005601 -0.0025771,0.0005071 -0.0042195,0.0017787 -0.00565,0.0005071 -0.0060738)))

utils/blocks.php

@@ -7,7 +7,7 @@ ini_set('memory_limit', '800M');
 
 $aCMDOptions
  = array(
-    "Manage service blocks / restrictions",
+    'Manage service blocks / restrictions',
     array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
     array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
     array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
@@ -29,8 +29,8 @@ if ($aResult['list']) {
 
     $aBlocks = getBucketBlocks();
     echo "\n";
-    printf(" %-40s | %12s | %7s | %13s | %31s | %8s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Block Time", "Sleeping");
+    printf(" %-40s | %12s | %7s | %13s | %31s | %8s\n", 'Key', 'Total Blocks', 'Current', 'Still Blocked', 'Last Block Time', 'Sleeping');
-    printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s-|-%'-8s\n", "", "", "", "", "", "");
+    printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s-|-%'-8s\n", '', '', '', '', '', '');
     foreach ($aBlocks as $sKey => $aDetails) {
         printf(
             " %-40s | %12s | %7s | %13s | %31s | %8s\n",
@@ -38,7 +38,7 @@ if ($aResult['list']) {
             $aDetails['totalBlocks'],
             (int)$aDetails['currentBucketSize'],
             $aDetails['currentlyBlocked']?'Y':'N',
-            date("r", $aDetails['lastBlockTimestamp']),
+            date('r', $aDetails['lastBlockTimestamp']),
             $aDetails['isSleeping']?'Y':'N'
         );
     }

utils/check_server_for_updates.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python
+
+import sys
+from osmium.replication import server
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print("Usage: python check_server_for_updates.py <server url> <sequence id>")
+        sys.exit(254)
+
+    seqid = int(sys.argv[2])
+
+    state = server.ReplicationServer(sys.argv[1]).get_state_info()
+
+    if state is None:
+        print("ERROR: Cannot get state from URL %s." % (sys.argv[1], ))
+        sys.exit(253)
+
+    if state.sequence <= seqid:
+        print("Database up to date.")
+        sys.exit(1)
+
+    print("New data available (%i => %i)." % (seqid, state.sequence))
+    sys.exit(0)

utils/country_languages.php

@@ -9,7 +9,7 @@ ini_set('display_errors', 'stderr');
 
 $aCMDOptions
  = array(
-    "Import country language data from osm wiki",
+    'Import country language data from osm wiki',
     array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
     array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
     array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
@@ -19,7 +19,7 @@ getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
 include(CONST_InstallPath.'/settings/phrase_settings.php');
 
 if (true) {
-    $sURL = 'http://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Country_Codes';
+    $sURL = 'https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Country_Codes';
     $sWikiPageXML = file_get_contents($sURL);
     if (preg_match_all('#\\| ([a-z]{2}) \\|\\| [^|]+\\|\\| ([a-z,]+)#', $sWikiPageXML, $aMatches, PREG_SET_ORDER)) {
         foreach ($aMatches as $aMatch) {

utils/importWikipedia.php

@@ -7,7 +7,7 @@ ini_set('memory_limit', '800M');
 
 $aCMDOptions
  = array(
-    "Create and setup nominatim search system",
+    'Create and setup nominatim search system',
     array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
     array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
     array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
@@ -379,7 +379,7 @@ if (isset($aCMDResult['link'])) {
 
         $sURL = $sNominatimBaseURL.'?format=xml&accept-language=en';
 
-        echo "\n-- ".$aRecord['name'].", ".$aRecord['infobox_type']."\n";
+        echo "\n-- ".$aRecord['name'].', '.$aRecord['infobox_type']."\n";
         $fMaxDist = 0.0000001;
         $bUnknown = false;
         switch (strtolower($aRecord['infobox_type'])) {
@@ -387,15 +387,15 @@ if (isset($aCMDResult['link'])) {
                 continue 2;
             case 'sea':
                 $fMaxDist = 60; // effectively turn it off
-                $sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
+                $sURL .= '&viewbox='.($aRecord['lon']-$fMaxDist).','.($aRecord['lat']+$fMaxDist).','.($aRecord['lon']+$fMaxDist).','.($aRecord['lat']-$fMaxDist);
                 break;
             case 'country':
             case 'island':
             case 'islands':
             case 'continent':
                 $fMaxDist = 60; // effectively turn it off
-                $sURL .= "&featuretype=country";
+                $sURL .= '&featuretype=country';
-                $sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
+                $sURL .= '&viewbox='.($aRecord['lon']-$fMaxDist).','.($aRecord['lat']+$fMaxDist).','.($aRecord['lon']+$fMaxDist).','.($aRecord['lat']-$fMaxDist);
                 break;
             case 'prefecture japan':
                 $aRecord['name'] = trim(str_replace(' Prefecture', ' ', $aRecord['name']));
@@ -415,14 +415,14 @@ if (isset($aCMDResult['link'])) {
             case '#australia state or territory':
             case 'russian federal subject':
                 $fMaxDist = 4;
-                $sURL .= "&featuretype=state";
+                $sURL .= '&featuretype=state';
-                $sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
+                $sURL .= '&viewbox='.($aRecord['lon']-$fMaxDist).','.($aRecord['lat']+$fMaxDist).','.($aRecord['lon']+$fMaxDist).','.($aRecord['lat']-$fMaxDist);
                 break;
             case 'protected area':
                 $fMaxDist = 1;
-                $sURL .= "&nearlat=".$aRecord['lat'];
+                $sURL .= '&nearlat='.$aRecord['lat'];
-                $sURL .= "&nearlon=".$aRecord['lon'];
+                $sURL .= '&nearlon='.$aRecord['lon'];
-                $sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
+                $sURL .= '&viewbox='.($aRecord['lon']-$fMaxDist).','.($aRecord['lat']+$fMaxDist).','.($aRecord['lon']+$fMaxDist).','.($aRecord['lat']-$fMaxDist);
                 break;
             case 'settlement':
                 $bUnknown = true;
@@ -444,8 +444,8 @@ if (isset($aCMDResult['link'])) {
             case 'russian city':
             case 'city':
                 $fMaxDist = 0.2;
-                $sURL .= "&featuretype=settlement";
+                $sURL .= '&featuretype=settlement';
-                $sURL .= "&viewbox=".($aRecord['lon']-0.5).",".($aRecord['lat']+0.5).",".($aRecord['lon']+0.5).",".($aRecord['lat']-0.5);
+                $sURL .= '&viewbox='.($aRecord['lon']-0.5).','.($aRecord['lat']+0.5).','.($aRecord['lon']+0.5).','.($aRecord['lat']-0.5);
                 break;
             case 'mountain':
             case 'mountain pass':
@@ -453,33 +453,33 @@ if (isset($aCMDResult['link'])) {
             case 'lake':
             case 'airport':
                 $fMaxDist = 0.2;
-                $sURL .= "&viewbox=".($aRecord['lon']-0.5).",".($aRecord['lat']+0.5).",".($aRecord['lon']+0.5).",".($aRecord['lat']-0.5);
+                $sURL .= '&viewbox='.($aRecord['lon']-0.5).','.($aRecord['lat']+0.5).','.($aRecord['lon']+0.5).','.($aRecord['lat']-0.5);
                 break;
             case 'ship begin':
                 $fMaxDist = 0.1;
                 $aTypes = array('wreck');
-                $sURL .= "&viewbox=".($aRecord['lon']-0.01).",".($aRecord['lat']+0.01).",".($aRecord['lon']+0.01).",".($aRecord['lat']-0.01);
+                $sURL .= '&viewbox='.($aRecord['lon']-0.01).','.($aRecord['lat']+0.01).','.($aRecord['lon']+0.01).','.($aRecord['lat']-0.01);
-                $sURL .= "&nearlat=".$aRecord['lat'];
+                $sURL .= '&nearlat='.$aRecord['lat'];
-                $sURL .= "&nearlon=".$aRecord['lon'];
+                $sURL .= '&nearlon='.$aRecord['lon'];
                 break;
             case 'road':
             case 'university':
             case 'company':
             case 'department':
                 $fMaxDist = 0.005;
-                $sURL .= "&viewbox=".($aRecord['lon']-0.01).",".($aRecord['lat']+0.01).",".($aRecord['lon']+0.01).",".($aRecord['lat']-0.01);
+                $sURL .= '&viewbox='.($aRecord['lon']-0.01).','.($aRecord['lat']+0.01).','.($aRecord['lon']+0.01).','.($aRecord['lat']-0.01);
-                $sURL .= "&bounded=1";
+                $sURL .= '&bounded=1';
-                $sURL .= "&nearlat=".$aRecord['lat'];
+                $sURL .= '&nearlat='.$aRecord['lat'];
-                $sURL .= "&nearlon=".$aRecord['lon'];
+                $sURL .= '&nearlon='.$aRecord['lon'];
                 break;
             default:
                 $bUnknown = true;
                 $fMaxDist = 0.005;
-                $sURL .= "&viewbox=".($aRecord['lon']-0.01).",".($aRecord['lat']+0.01).",".($aRecord['lon']+0.01).",".($aRecord['lat']-0.01);
+                $sURL .= '&viewbox='.($aRecord['lon']-0.01).','.($aRecord['lat']+0.01).','.($aRecord['lon']+0.01).','.($aRecord['lat']-0.01);
                 // $sURL .= "&bounded=1";
-                $sURL .= "&nearlat=".$aRecord['lat'];
+                $sURL .= '&nearlat='.$aRecord['lat'];
-                $sURL .= "&nearlon=".$aRecord['lon'];
+                $sURL .= '&nearlon='.$aRecord['lon'];
-                echo "-- Unknown: ".$aRecord['infobox_type']."\n";
+                echo '-- Unknown: '.$aRecord['infobox_type']."\n";
                 break;
         }
         $sNameURL = $sURL.'&q='.urlencode($aRecord['name']);
@@ -526,14 +526,14 @@ if (isset($aCMDResult['link'])) {
                 elseif ($iRank <= 26) $fMaxDist = 0.001;
                 else $fMaxDist = 0.001;
             }
-            echo "-- FOUND \"".substr($aNominatRecords[$i]['DISPLAY_NAME'], 0, 50);
+            echo '-- FOUND "'.substr($aNominatRecords[$i]['DISPLAY_NAME'], 0, 50);
-            echo "\", ".$aNominatRecords[$i]['CLASS'].", ".$aNominatRecords[$i]['TYPE'];
+            echo '", '.$aNominatRecords[$i]['CLASS'].', '.$aNominatRecords[$i]['TYPE'];
-            echo ", ".$aNominatRecords[$i]['PLACE_RANK'].", ".$aNominatRecords[$i]['OSM_TYPE'];
+            echo ', '.$aNominatRecords[$i]['PLACE_RANK'].', '.$aNominatRecords[$i]['OSM_TYPE'];
             echo " (dist:$fDiff, max:$fMaxDist)\n";
             if ($fDiff > $fMaxDist) {
                 echo "-- Diff too big $fDiff (max: $fMaxDist)".$aRecord['lat'].','.$aNominatRecords[$i]['LAT'].' & '.$aRecord['lon'].','.$aNominatRecords[$i]['LON']." \n";
             } else {
-                $sSQL = "update wikipedia_article set osm_type=";
+                $sSQL = 'update wikipedia_article set osm_type=';
                 switch ($aNominatRecords[$i]['OSM_TYPE']) {
                     case 'relation':
                         $sSQL .= "'R'";
@@ -545,7 +545,7 @@ if (isset($aCMDResult['link'])) {
                         $sSQL .= "'N'";
                         break;
                 }
-                $sSQL .= ", osm_id=".$aNominatRecords[$i]['OSM_ID']." where language = '".pg_escape_string($aRecord['language'])."' and title = '".pg_escape_string($aRecord['title'])."'";
+                $sSQL .= ', osm_id='.$aNominatRecords[$i]['OSM_ID']." where language = '".pg_escape_string($aRecord['language'])."' and title = '".pg_escape_string($aRecord['title'])."'";
                 $oDB->query($sSQL);
                 break;
             }

utils/imports.php

@@ -7,7 +7,7 @@ ini_set('memory_limit', '800M');
 
 $aCMDOptions
  = array(
-    "Create and setup nominatim search system",
+    'Create and setup nominatim search system',
     array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
     array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
     array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
@@ -28,20 +28,24 @@ if (isset($aCMDResult['parse-tiger'])) {
         set_time_limit(30);
         preg_match('#([0-9]{5})_(.*)#', basename($sImportFile), $aMatch);
         $sCountyID = $aMatch[1];
-        echo "Processing ".$sCountyID."...\n";
+
+        echo 'Processing '.$sCountyID."...\n";
         $sUnzipCmd = "unzip -d $sTempDir $sImportFile";
         exec($sUnzipCmd);
-        $sShapeFile = $sTempDir.'/'.basename($sImportFile, '.zip').'.shp';
+
-        if (!file_exists($sShapeFile)) {
+        $sShapeFilename = $sTempDir.'/'.basename($sImportFile, '.zip').'.shp';
+        $sSqlFilenameTmp = $sTempDir.'/'.$sCountyID.'.sql';
+        $sSqlFilename = CONST_Tiger_Data_Path.'/'.$sCountyID.'.sql';
+
+        if (!file_exists($sShapeFilename)) {
             echo "Failed unzip ($sImportFile)\n";
         } else {
-            $sParseCmd = CONST_BasePath.'/utils/tigerAddressImport.py '.$sShapeFile;
+            $sParseCmd = CONST_BasePath.'/utils/tigerAddressImport.py '.$sShapeFilename.' '.$sSqlFilenameTmp;
             exec($sParseCmd);
-            $sOsmFile = $sTempDir.'/'.basename($sImportFile, '.zip').'.osm1.osm';
+            if (!file_exists($sSqlFilenameTmp)) {
-            if (!file_exists($sOsmFile)) {
                 echo "Failed parse ($sImportFile)\n";
             } else {
-                copy($sOsmFile, CONST_Tiger_Data_Path.'/'.$sCountyID.'.sql');
+                copy($sSqlFilenameTmp, $sSqlFilename);
             }
         }
         // Cleanup

utils/query.php

@@ -9,7 +9,7 @@ ini_set('memory_limit', '800M');
 
 $aCMDOptions
 = array(
-   "Query database from command line. Returns search result as JSON.",
+   'Query database from command line. Returns search result as JSON.',
    array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
    array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
    array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
@@ -41,7 +41,7 @@ if ($oParams->getBool('search')) {
 
     $aSearchResults = $oGeocode->lookup();
 
-    if (version_compare(phpversion(), "5.4.0", '<')) {
+    if (version_compare(phpversion(), '5.4.0', '<')) {
         echo json_encode($aSearchResults);
     } else {
         echo json_encode($aSearchResults, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)."\n";

utils/server_compare.php

@@ -1,7 +1,7 @@
 #!/usr/bin/php -Cq
 <?php
 
-$sFile = "sample.log.txt"; // Apache log file
+$sFile = 'sample.log.txt'; // Apache log file
 $sHost1 = 'http://mq-open-search-lm02.ihost.aol.com:8000/nominatim/v1';
 $sHost2 = 'http://mq-open-search-lm03.ihost.aol.com:8000/nominatim/v1';
 
@@ -11,7 +11,7 @@ $sHost2Escaped = str_replace('/', '\\/', $sHost2);
 
 $aToDo = array(251, 293, 328, 399.1, 455.1, 479, 496, 499, 574, 609, 702, 790, 846, 865, 878, 894, 902, 961, 980);
 
-$hFile = @fopen($sFile, "r");
+$hFile = @fopen($sFile, 'r');
 if (!$hFile) {
     echo "Unable to open file: $sFile\n";
     exit;

utils/setup.php

@@ -7,7 +7,7 @@ ini_set('memory_limit', '800M');
 
 $aCMDOptions
 = array(
-   "Create and setup nominatim search system",
+   'Create and setup nominatim search system',
    array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
    array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
    array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
@@ -28,7 +28,7 @@ $aCMDOptions
    array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
    array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
    array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
-   array('no-partitions', '', 0, 1, 0, 0, 'bool', "Do not partition search indices (speeds up import of single country extracts)"),
+   array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
    array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
    array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
    array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
@@ -65,11 +65,11 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) {
 $iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
 if ($iInstances < 1) {
     $iInstances = 1;
-    echo "WARNING: resetting threads to $iInstances\n";
+    warn("resetting threads to $iInstances");
 }
 if ($iInstances > getProcessorCount()) {
     $iInstances = getProcessorCount();
-    echo "WARNING: resetting threads to $iInstances\n";
+    warn("resetting threads to $iInstances");
 }
 
 // Assume we can steal all the cache memory in the box (unless told otherwise)
@@ -83,7 +83,7 @@ $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 
 if ($aCMDResult['create-db'] || $aCMDResult['all']) {
-    echo "Create DB\n";
+    info('Create DB');
     $bDidSomething = true;
     $oDB = DB::connect(CONST_Database_DSN, false);
     if (!PEAR::isError($oDB)) {
@@ -93,18 +93,16 @@ if ($aCMDResult['create-db'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
-    echo "Setup DB\n";
+    info('Setup DB');
     $bDidSomething = true;
 
-    // TODO: path detection, detection memory, etc.
-    //
     $oDB =& getDB();
 
     $fPostgresVersion = getPostgresVersion($oDB);
     echo 'Postgres version found: '.$fPostgresVersion."\n";
 
     if ($fPostgresVersion < 9.1) {
-        fail("Minimum supported version of Postgresql is 9.1.");
+        fail('Minimum supported version of Postgresql is 9.1.');
     }
 
     pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
@@ -117,7 +115,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
 
     if ($iNumFunc == 0) {
         pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable");
-        echo "WARNING: Postgresql is too old. extratags and namedetails API not available.";
+        warn('Postgresql is too old. extratags and namedetails API not available.');
     }
 
     $fPostgisVersion = getPostgisVersion($oDB);
@@ -132,9 +130,29 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
         pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
     }
 
+    $i = chksql($oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'"));
+    if ($i == 0) {
+        echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
+        echo "\n          createuser ".CONST_Database_Web_User."\n\n";
+        exit(1);
+    }
+
+    // Try accessing the C module, so we know early if something is wrong
+    // and can simply error out.
+    $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '";
+    $sSQL .= CONST_InstallPath."/module/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT";
+    $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);';
+    $oResult = $oDB->query($sSQL);
+
+    if (PEAR::isError($oResult)) {
+        echo "\nERROR: Failed to load nominatim module. Reason:\n";
+        echo $oResult->userinfo."\n\n";
+        exit(1);
+    }
+
     if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) {
-        echo "Error: you need to download the country_osm_grid first:";
+        echo 'Error: you need to download the country_osm_grid first:';
-        echo "\n    wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz http://www.nominatim.org/data/country_grid.sql.gz\n";
+        echo "\n    wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n";
         exit(1);
     }
 
@@ -145,7 +163,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
     if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) {
         pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
     } else {
-        echo "WARNING: external UK postcode table not found.\n";
+        warn('external UK postcode table not found.');
     }
     if (CONST_Use_Extra_US_Postcodes) {
         pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
@@ -164,7 +182,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['import-data'] || $aCMDResult['all']) {
-    echo "Import\n";
+    info('Import data');
     $bDidSomething = true;
 
     $osm2pgsql = CONST_Osm2pgsql_Binary;
@@ -198,16 +216,18 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
-    echo "Functions\n";
+    info('Create Functions');
     $bDidSomething = true;
-    if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built");
+    if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) {
+        fail('nominatim module not built');
+    }
     create_sql_functions($aCMDResult);
 }
 
 if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
+    info('Create Tables');
     $bDidSomething = true;
 
-    echo "Tables\n";
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
     $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
     $sTemplate = replace_tablespace(
@@ -243,12 +263,12 @@ if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
     pgsqlRunScript($sTemplate, false);
 
     // re-run the functions
-    echo "Functions\n";
+    info('Recreate Functions');
     create_sql_functions($aCMDResult);
 }
 
 if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
-    echo "Partition Tables\n";
+    info('Create Partition Tables');
     $bDidSomething = true;
 
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
@@ -288,7 +308,7 @@ if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
 
 
 if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
-    echo "Partition Functions\n";
+    info('Create Partition Functions');
     $bDidSomething = true;
 
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
@@ -301,24 +321,22 @@ if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
     $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
     $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
     if (file_exists($sWikiArticlesFile)) {
-        echo "Importing wikipedia articles...";
+        info('Importing wikipedia articles');
         pgsqlRunDropAndRestore($sWikiArticlesFile);
-        echo "...done\n";
     } else {
-        echo "WARNING: wikipedia article dump file not found - places will have default importance\n";
+        warn('wikipedia article dump file not found - places will have default importance');
     }
     if (file_exists($sWikiRedirectsFile)) {
-        echo "Importing wikipedia redirects...";
+        info('Importing wikipedia redirects');
         pgsqlRunDropAndRestore($sWikiRedirectsFile);
-        echo "...done\n";
     } else {
-        echo "WARNING: wikipedia redirect dump file not found - some place importance values may be missing\n";
+        warn('wikipedia redirect dump file not found - some place importance values may be missing');
     }
 }
 
 
 if ($aCMDResult['load-data'] || $aCMDResult['all']) {
-    echo "Drop old Data\n";
+    info('Drop old Data');
     $bDidSomething = true;
 
     $oDB =& getDB();
@@ -361,11 +379,11 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
 
     // pre-create the word list
     if (!$aCMDResult['disable-token-precalc']) {
-        echo "Loading word list\n";
+        info('Loading word list');
         pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
     }
 
-    echo "Load Data\n";
+    info('Load Data');
     $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry';
 
     $aDBInstances = array();
@@ -375,7 +393,7 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
         $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
         $sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
         $sSQL .= "          and ST_GeometryType(geometry) = 'ST_LineString')";
-        $sSQL .= " and ST_IsValid(geometry)";
+        $sSQL .= ' and ST_IsValid(geometry)';
         if ($aCMDResult['verbose']) echo "$sSQL\n";
         if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
             fail(pg_last_error($aDBInstances[$i]->connection));
@@ -402,13 +420,13 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
         echo '.';
     }
     echo "\n";
-    echo "Reanalysing database...\n";
+    info('Reanalysing database');
     pgsqlRunScript('ANALYSE');
 
     $sDatabaseDate = getDatabaseDate($oDB);
     pg_query($oDB->connection, 'TRUNCATE import_status');
     if ($sDatabaseDate === false) {
-        echo "WARNING: could not determine database date.\n";
+        warn('could not determine database date.');
     } else {
         $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
         pg_query($oDB->connection, $sSQL);
@@ -417,6 +435,7 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['import-tiger-data']) {
+    info('Import Tiger data');
     $bDidSomething = true;
 
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
@@ -440,7 +459,7 @@ if ($aCMDResult['import-tiger-data']) {
 
     foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) {
         echo $sFile.': ';
-        $hFile = fopen($sFile, "r");
+        $hFile = fopen($sFile, 'r');
         $sSQL = fgets($hFile, 100000);
         $iLines = 0;
 
@@ -453,7 +472,7 @@ if ($aCMDResult['import-tiger-data']) {
                     if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
                     $iLines++;
                     if ($iLines == 1000) {
-                        echo ".";
+                        echo '.';
                         $iLines = 0;
                     }
                 }
@@ -474,7 +493,7 @@ if ($aCMDResult['import-tiger-data']) {
         echo "\n";
     }
 
-    echo "Creating indexes\n";
+    info('Creating indexes on Tiger data');
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
     $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
     $sTemplate = replace_tablespace(
@@ -491,26 +510,61 @@ if ($aCMDResult['import-tiger-data']) {
 }
 
 if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
+    info('Calculate Postcodes');
     $bDidSomething = true;
     $oDB =& getDB();
-    if (!pg_query($oDB->connection, 'DELETE from placex where osm_type=\'P\'')) fail(pg_last_error($oDB->connection));
+    if (!pg_query($oDB->connection, 'TRUNCATE location_postcode')) {
-    $sSQL = "insert into placex (osm_type,osm_id,class,type,address,country_code,geometry) ";
+        fail(pg_last_error($oDB->connection));
-    $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',";
+    }
-    $sSQL .= "hstore('postcode', pc),country_code,";
+
-    $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from (select country_code,";
+    $sSQL  = 'INSERT INTO location_postcode';
-    $sSQL .= "address->'postcode' as pc,";
+    $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
-    $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
+    $sSQL .= "SELECT nextval('seq_place'), 1, country_code,";
-    $sSQL .= "from placex where address ? 'postcode' group by country_code,pc) as x ";
+    $sSQL .= "       upper(trim (both ' ' from address->'postcode')) as pc,";
-    $sSQL .= "where ST_Point(x,y) is not null";
+    $sSQL .= '       ST_Centroid(ST_Collect(ST_Centroid(geometry)))';
-    if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
+    $sSQL .= '  FROM placex';
+    $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
+    $sSQL .= '       AND geometry IS NOT null';
+    $sSQL .= ' GROUP BY country_code, pc';
+
+    if (!pg_query($oDB->connection, $sSQL)) {
+        fail(pg_last_error($oDB->connection));
+    }
 
     if (CONST_Use_Extra_US_Postcodes) {
-        $sSQL = "insert into placex (osm_type,osm_id,class,type,address,country_code,geometry) ";
+        // only add postcodes that are not yet available in OSM
-        $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',";
+        $sSQL  = 'INSERT INTO location_postcode';
-        $sSQL .= "hstore('postcode', postcode),'us',";
+        $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
-        $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
+        $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
+        $sSQL .= '       ST_SetSRID(ST_Point(x,y),4326)';
+        $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
+        $sSQL .= '        (SELECT postcode FROM location_postcode';
+        $sSQL .= "          WHERE country_code = 'us')";
         if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
     }
+
+    // add missing postcodes for GB (if available)
+    $sSQL  = 'INSERT INTO location_postcode';
+    $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
+    $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry";
+    $sSQL .= '  FROM gb_postcode WHERE postcode NOT IN';
+    $sSQL .= '           (SELECT postcode FROM location_postcode';
+    $sSQL .= "             WHERE country_code = 'gb')";
+    if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
+
+    if (!$aCMDResult['all']) {
+        $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
+        $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
+        if (!pg_query($oDB->connection, $sSQL)) {
+            fail(pg_last_error($oDB->connection));
+        }
+    }
+    $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
+    $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
+
+    if (!pg_query($oDB->connection, $sSQL)) {
+        fail(pg_last_error($oDB->connection));
+    }
 }
 
 if ($aCMDResult['osmosis-init']) {
@@ -522,15 +576,23 @@ if ($aCMDResult['index'] || $aCMDResult['all']) {
     $bDidSomething = true;
     $sOutputFile = '';
     $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
+    info('Index ranks 0 - 4');
     passthruCheckReturn($sBaseCmd.' -R 4');
     if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
+    info('Index ranks 5 - 25');
     passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
     if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
+    info('Index ranks 26 - 30');
     passthruCheckReturn($sBaseCmd.' -r 26');
+
+    info('Index postcodes');
+    $oDB =& getDB();
+    $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
+    if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
 }
 
 if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
-    echo "Search indices\n";
+    info('Create Search indices');
     $bDidSomething = true;
 
     $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
@@ -554,12 +616,12 @@ if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
-    echo 'Creating search index for default country names';
+    info('Create search index for default country names');
     $bDidSomething = true;
 
     pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')");
     pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')");
-    pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x");
+    pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x');
     pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x");
 
     $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v), country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k ';
@@ -580,6 +642,7 @@ if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
 }
 
 if ($aCMDResult['drop']) {
+    info('Drop tables only required for updates');
     // The implementation is potentially a bit dangerous because it uses
     // a positive selection of tables to keep, and deletes everything else.
     // Including any tables that the unsuspecting user might have manually
@@ -588,21 +651,21 @@ if ($aCMDResult['drop']) {
 
     // tables we want to keep. everything else goes.
     $aKeepTables = array(
-                    "*columns",
+                    '*columns',
-                    "import_polygon_*",
+                    'import_polygon_*',
-                    "import_status",
+                    'import_status',
-                    "place_addressline",
+                    'place_addressline',
-                    "location_property*",
+                    'location_postcode',
-                    "placex",
+                    'location_property*',
-                    "search_name",
+                    'placex',
-                    "seq_*",
+                    'search_name',
-                    "word",
+                    'seq_*',
-                    "query_log",
+                    'word',
-                    "new_query_log",
+                    'query_log',
-                    "gb_postcode",
+                    'new_query_log',
-                    "spatial_ref_sys",
+                    'spatial_ref_sys',
-                    "country_name",
+                    'country_name',
-                    "place_classtype_*"
+                    'place_classtype_*'
                    );
 
     $oDB =& getDB();
@@ -628,7 +691,7 @@ if ($aCMDResult['drop']) {
     }
 
     if (!is_null(CONST_Osm2pgsql_Flatnode_File)) {
-        if ($aCMDResult['verbose']) echo "deleting ".CONST_Osm2pgsql_Flatnode_File."\n";
+        if ($aCMDResult['verbose']) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n";
         unlink(CONST_Osm2pgsql_Flatnode_File);
     }
 }
@@ -636,18 +699,25 @@ if ($aCMDResult['drop']) {
 if (!$bDidSomething) {
     showUsage($aCMDOptions, true);
 } else {
-    echo "Setup finished.\n";
+    echo "Summary of warnings:\n\n";
+    repeatWarnings();
+    echo "\n";
+    info('Setup finished.');
 }
 
 
 function pgsqlRunScriptFile($sFilename)
 {
+    global $aCMDResult;
     if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
 
     // Convert database DSN to psql parameters
     $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
     if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
     $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
+    if (!$aCMDResult['verbose']) {
+        $sCMD .= ' -q';
+    }
 
     $ahGzipPipes = null;
     if (preg_match('/\\.gz$/', $sFilename)) {
@@ -694,31 +764,12 @@ function pgsqlRunScriptFile($sFilename)
 function pgsqlRunScript($sScript, $bfatal = true)
 {
     global $aCMDResult;
-    // Convert database DSN to psql parameters
+    runSQLScript(
-    $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
+        $sScript,
-    if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
+        $bfatal,
-    $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
+        $aCMDResult['verbose'],
-    if ($bfatal && !$aCMDResult['ignore-errors'])
+        $aCMDResult['ignore-errors']
-        $sCMD .= ' -v ON_ERROR_STOP=1';
+    );
-    $aDescriptors = array(
-                     0 => array('pipe', 'r'),
-                     1 => STDOUT,
-                     2 => STDERR
-                    );
-    $ahPipes = null;
-    $hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes);
-    if (!is_resource($hProcess)) fail('unable to start pgsql');
-
-    while (strlen($sScript)) {
-        $written = fwrite($ahPipes[0], $sScript);
-        if ($written <= 0) break;
-        $sScript = substr($sScript, $written);
-    }
-    fclose($ahPipes[0]);
-    $iReturn = proc_close($hProcess);
-    if ($bfatal && $iReturn > 0) {
-        fail("pgsql returned with error code ($iReturn)");
-    }
 }
 
 function pgsqlRunPartitionScript($sTemplate)

utils/specialphrases.php

@@ -8,7 +8,7 @@ ini_set('display_errors', 'stderr');
 
 $aCMDOptions
 = array(
-   "Import and export special phrases",
+   'Import and export special phrases',
    array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
    array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
    array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
@@ -27,7 +27,7 @@ if ($aCMDResult['wiki-import']) {
          'ia,is,it,ja,mk,nl,no,pl,ps,pt,ru,sk,sl,sv,uk,vi');
 
     foreach (explode(',', $sLanguageIn) as $sLanguage) {
-        $sURL = 'http://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/'.strtoupper($sLanguage);
+        $sURL = 'https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/'.strtoupper($sLanguage);
         $sWikiPageXML = file_get_contents($sURL);
         if (preg_match_all('#\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([\\-YN])#', $sWikiPageXML, $aMatches, PREG_SET_ORDER)) {
             foreach ($aMatches as $aMatch) {
@@ -76,30 +76,30 @@ if ($aCMDResult['wiki-import']) {
         }
     }
 
-    echo "create index idx_placex_classtype on placex (class, type);";
+    echo 'create index idx_placex_classtype on placex (class, type);';
 
     foreach ($aPairs as $aPair) {
-        echo "create table place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1]);
+        echo 'create table place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]);
         if (CONST_Tablespace_Aux_Data)
-            echo " tablespace ".CONST_Tablespace_Aux_Data;
+            echo ' tablespace '.CONST_Tablespace_Aux_Data;
-        echo " as select place_id as place_id,st_centroid(geometry) as centroid from placex where ";
+        echo ' as select place_id as place_id,st_centroid(geometry) as centroid from placex where ';
         echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."'";
         echo ";\n";
 
-        echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_centroid ";
+        echo 'CREATE INDEX idx_place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).'_centroid ';
-        echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING GIST (centroid)";
+        echo 'ON place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).' USING GIST (centroid)';
         if (CONST_Tablespace_Aux_Index)
-            echo " tablespace ".CONST_Tablespace_Aux_Index;
+            echo ' tablespace '.CONST_Tablespace_Aux_Index;
         echo ";\n";
 
-        echo "CREATE INDEX idx_place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])."_place_id ";
+        echo 'CREATE INDEX idx_place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).'_place_id ';
-        echo "ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1])." USING btree(place_id)";
+        echo 'ON place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).' USING btree(place_id)';
         if (CONST_Tablespace_Aux_Index)
-            echo " tablespace ".CONST_Tablespace_Aux_Index;
+            echo ' tablespace '.CONST_Tablespace_Aux_Index;
         echo ";\n";
 
-        echo "GRANT SELECT ON place_classtype_".pg_escape_string($aPair[0])."_".pg_escape_string($aPair[1]).' TO "'.CONST_Database_Web_User."\";\n";
+        echo 'GRANT SELECT ON place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).' TO "'.CONST_Database_Web_User."\";\n";
     }
 
-    echo "drop index idx_placex_classtype;";
+    echo 'drop index idx_placex_classtype;';
 }

utils/update.php

@@ -7,18 +7,18 @@ ini_set('memory_limit', '800M');
 
 $aCMDOptions
 = array(
-   "Import / update / index osm data",
+   'Import / update / index osm data',
    array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
    array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
    array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
 
    array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
+   array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
    array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
    array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
-   array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),
    array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
 
-   array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
+   array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
 
    array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
    array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
@@ -34,6 +34,8 @@ $aCMDOptions
    array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
 
    array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
+   array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
+   array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
   );
 getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
 
@@ -59,6 +61,13 @@ if (!is_null(CONST_Osm2pgsql_Flatnode_File)) {
 }
 
 if ($aResult['init-updates']) {
+    // sanity check that the replication URL is correct
+    $sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt');
+    if ($sBaseState === false) {
+        echo "\nCannot find state.txt file at the configured replication URL.\n";
+        echo "Does the URL point to a directory containing OSM update data?\n\n";
+        fail('replication URL not reachable.');
+    }
     $sSetup = CONST_InstallPath.'/utils/setup.php';
     $iRet = -1;
     passthru($sSetup.' --create-functions --enable-diff-updates', $iRet);
@@ -68,32 +77,43 @@ if ($aResult['init-updates']) {
 
     $sDatabaseDate = getDatabaseDate($oDB);
     if ($sDatabaseDate === false) {
-        fail("Cannot determine date of database.");
+        fail('Cannot determine date of database.');
     }
-    $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ',
+    $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60));
-                          strtotime($sDatabaseDate) - (3*60*60));
 
     // get the appropriate state id
     $aOutput = 0;
-    exec(CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url,
+    $sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url;
-        $aOutput, $iRet);
+    exec($sCmd, $aOutput, $iRet);
-    if ($iRet != 0) {
+    if ($iRet != 0 || $aOutput[0] == 'None') {
         fail('Error running pyosmium tools');
     }
 
     pg_query($oDB->connection, 'TRUNCATE import_status');
     $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('";
-    $sSQL .= $sDatabaseDate."',".$aOutput[0].", true)";
+    $sSQL .= $sDatabaseDate."',".$aOutput[0].', true)';
     if (!pg_query($oDB->connection, $sSQL)) {
-        fail("Could not enter sequence into database.");
+        fail('Could not enter sequence into database.');
     }
 
     echo "Done. Database updates will start at sequence $aOutput[0] ($sWindBack)\n";
 }
 
+if ($aResult['check-for-updates']) {
+    $aLastState = chksql($oDB->getRow('SELECT sequence_id FROM import_status'));
+
+    if (!$aLastState['sequence_id']) {
+        fail('Updates not set up. Please run ./utils/update.php --init-updates.');
+    }
+
+    system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet);
+    exit($iRet);
+}
+
 if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
     // import diffs and files directly (e.g. from osmosis --rri)
     $sNextFile = isset($aResult['import-diff']) ? $aResult['import-diff'] : $aResult['import-file'];
+
     if (!file_exists($sNextFile)) {
         fail("Cannot open $sNextFile\n");
     }
@@ -110,31 +130,37 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
     // Don't update the import status - we don't know what this file contains
 }
 
+if ($aResult['calculate-postcodes']) {
+    info('Update postcodes centroids');
+    $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql');
+    runSQLScript($sTemplate, true, true);
+}
+
 $sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
 $bHaveDiff = false;
 $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
 $sContentURL = '';
 if (isset($aResult['import-node']) && $aResult['import-node']) {
     if ($bUseOSMApi) {
-        $sContentURL = 'http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
+        $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
     } else {
-        $sContentURL = 'http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
     }
 }
 
 if (isset($aResult['import-way']) && $aResult['import-way']) {
     if ($bUseOSMApi) {
-        $sContentURL = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
+        $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
     } else {
-        $sContentURL = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
     }
 }
 
 if (isset($aResult['import-relation']) && $aResult['import-relation']) {
     if ($bUseOSMApi) {
-        $sContentURLsModifyXMLstr = 'http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
+        $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
     } else {
-        $sContentURL = 'http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
+        $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
     }
 }
 
@@ -157,7 +183,7 @@ if ($aResult['deduplicate']) {
     $oDB =& getDB();
 
     if (getPostgresVersion($oDB) < 9.3) {
-        fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
+        fail('ERROR: deduplicate is only currently supported in postgresql 9.3');
     }
 
     $sSQL = 'select partition from country_name order by country_code';
@@ -166,7 +192,7 @@ if ($aResult['deduplicate']) {
 
     // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates
     foreach ($aPartitions as $i => $sPartition) {
-        $sSQL = "select count(*) from search_name_".$sPartition;
+        $sSQL = 'select count(*) from search_name_'.$sPartition;
         $nEntries = chksql($oDB->getOne($sSQL));
         if ($nEntries == 0) {
             unset($aPartitions[$i]);
@@ -174,14 +200,14 @@ if ($aResult['deduplicate']) {
     }
 
     $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' '";
-    $sSQL .= " and class is null and type is null and country_code is null";
+    $sSQL .= ' and class is null and type is null and country_code is null';
-    $sSQL .= " group by word_token having count(*) > 1 order by word_token";
+    $sSQL .= ' group by word_token having count(*) > 1 order by word_token';
     $aDuplicateTokens = chksql($oDB->getAll($sSQL));
     foreach ($aDuplicateTokens as $aToken) {
         if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue;
-        echo "Deduping ".$aToken['word_token']."\n";
+        echo 'Deduping '.$aToken['word_token']."\n";
-        $sSQL = "select word_id,";
+        $sSQL = 'select word_id,';
-        $sSQL .= " (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num";
+        $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num';
         $sSQL .= " from word where word_token = '".$aToken['word_token'];
         $sSQL .= "' and class is null and type is null and country_code is null order by num desc";
         $aTokenSet = chksql($oDB->getAll($sSQL));
@@ -190,40 +216,46 @@ if ($aResult['deduplicate']) {
         $iKeepID = $aKeep['word_id'];
 
         foreach ($aTokenSet as $aRemove) {
-            $sSQL = "update search_name set";
+            $sSQL = 'update search_name set';
-            $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID."),";
+            $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),';
-            $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
+            $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
-            $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
+            $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
             chksql($oDB->query($sSQL));
 
-            $sSQL = "update search_name set";
+            $sSQL = 'update search_name set';
-            $sSQL .= " nameaddress_vector = array_replace(nameaddress_vector,".$aRemove['word_id'].",".$iKeepID.")";
+            $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')';
-            $sSQL .= " where nameaddress_vector @> ARRAY[".$aRemove['word_id']."]";
+            $sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']';
             chksql($oDB->query($sSQL));
 
-            $sSQL = "update location_area_country set";
+            $sSQL = 'update location_area_country set';
-            $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
+            $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
-            $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
+            $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
             chksql($oDB->query($sSQL));
 
             foreach ($aPartitions as $sPartition) {
-                $sSQL = "update search_name_".$sPartition." set";
+                $sSQL = 'update search_name_'.$sPartition.' set';
-                $sSQL .= " name_vector = array_replace(name_vector,".$aRemove['word_id'].",".$iKeepID.")";
+                $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')';
-                $sSQL .= " where name_vector @> ARRAY[".$aRemove['word_id']."]";
+                $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']';
                 chksql($oDB->query($sSQL));
 
-                $sSQL = "update location_area_country set";
+                $sSQL = 'update location_area_country set';
-                $sSQL .= " keywords = array_replace(keywords,".$aRemove['word_id'].",".$iKeepID.")";
+                $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')';
-                $sSQL .= " where keywords @> ARRAY[".$aRemove['word_id']."]";
+                $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']';
                 chksql($oDB->query($sSQL));
             }
 
-            $sSQL = "delete from word where word_id = ".$aRemove['word_id'];
+            $sSQL = 'delete from word where word_id = '.$aRemove['word_id'];
             chksql($oDB->query($sSQL));
         }
     }
 }
 
+if ($aResult['recompute-word-counts']) {
+    info('Recompute frequency of full-word search terms');
+    $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql');
+    runSQLScript($sTemplate, true, true);
+}
+
 if ($aResult['index']) {
     passthru(CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
 }
@@ -231,7 +263,7 @@ if ($aResult['index']) {
 if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
     //
     if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
-        fail("Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n");
+        fail("Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");
     }
 
     $sImportFile = CONST_InstallPath.'/osmosischange.osc';
@@ -274,7 +306,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                 if ($iResult == 3) {
                     echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n";
                     sleep(CONST_Replication_Recheck_Interval);
-                } else if ($iResult != 0) {
+                } elseif ($iResult != 0) {
                     echo 'ERROR: updates failed.';
                     exit($iResult);
                 } else {
@@ -311,7 +343,11 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
 
             // write the update logs
             $iFileSize = filesize($sImportFile);
-            $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
+            $sSQL = 'INSERT INTO import_osmosis_log';
+            $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
+            $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
+            $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
+            $sSQL .= date('Y-m-d H:i:s')."','import')";
             var_Dump($sSQL);
             chksql($oDB->query($sSQL));
 
@@ -334,12 +370,16 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
                 exit($iErrorLevel);
             }
 
-            $sSQL = "INSERT INTO import_osmosis_log (batchend, batchseq, batchsize, starttime, endtime, event) values ('$sBatchEnd',$iEndSequence,$iFileSize,'".date('Y-m-d H:i:s', $fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
+            $sSQL = 'INSERT INTO import_osmosis_log';
+            $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
+            $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
+            $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','";
+            $sSQL .= date('Y-m-d H:i:s')."','index')";
             var_Dump($sSQL);
             $oDB->query($sSQL);
             echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n";
 
-            $sSQL = "update import_status set indexed = true";
+            $sSQL = 'update import_status set indexed = true';
             $oDB->query($sSQL);
         }
 
@@ -348,4 +388,3 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
         if (!$aResult['import-osmosis-all']) exit(0);
     }
 }
-

utils/warm.php

@@ -6,7 +6,7 @@ require_once(CONST_BasePath.'/lib/init-cmd.php');
 ini_set('memory_limit', '800M');
 
 $aCMDOptions = array(
-                "Tools to warm nominatim db",
+                'Tools to warm nominatim db',
                 array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
                 array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
                 array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
@@ -31,7 +31,7 @@ if (!$aResult['search-only']) {
     $oPlaceLookup->setIncludeAddressDetails(true);
     $oPlaceLookup->setLanguagePreference(array('en'));
 
-    echo "Warm reverse: ";
+    echo 'Warm reverse: ';
     if ($bVerbose) echo "\n";
     for ($i = 0; $i < 1000; $i++) {
         $fLat = rand(-9000, 9000) / 100;
@@ -46,7 +46,7 @@ if (!$aResult['search-only']) {
             );
             if ($bVerbose) echo $aDetails['langaddress']."\n";
         } else {
-            echo ".";
+            echo '.';
         }
     }
     echo "\n";
@@ -55,7 +55,7 @@ if (!$aResult['search-only']) {
 if (!$aResult['reverse-only']) {
     $oGeocode = new Nominatim\Geocode($oDB);
 
-    echo "Warm search: ";
+    echo 'Warm search: ';
     if ($bVerbose) echo "\n";
     $sSQL = 'select word from word where word is not null order by search_name_count desc limit 1000';
     foreach ($oDB->getCol($sSQL) as $sWord) {
@@ -64,6 +64,6 @@ if (!$aResult['reverse-only']) {
         $oGeocode->setQuery($sWord);
         $aSearchResults = $oGeocode->lookup();
         if ($bVerbose) echo $aSearchResults[0]['langaddress']."\n";
-        else echo ".";
+        else echo '.';
     }
 }

vagrant/Install-on-Centos-7.sh

@@ -19,18 +19,21 @@
 
 # Now you can install all packages needed for Nominatim:
 
-    sudo yum install -y postgresql-server postgresql-contrib postgresql-devel postgis postgis-utils \
+#DOCS:    :::sh
+    sudo yum install -y postgresql-server postgresql-contrib postgresql-devel \
+                        postgis postgis-utils \
                         git cmake make gcc gcc-c++ libtool policycoreutils-python \
-                        php-pgsql php php-pear php-pear-DB php-intl libpqxx-devel proj-epsg \
+                        php-pgsql php php-pear php-pear-DB php-intl libpqxx-devel \
-                        bzip2-devel proj-devel geos-devel libxml2-devel boost-devel expat-devel zlib-devel
+                        proj-epsg bzip2-devel proj-devel libxml2-devel boost-devel \
+                        expat-devel zlib-devel
 
 # If you want to run the test suite, you need to install the following
 # additional packages:
 
-    sudo yum install -y python-pip python-Levenshtein python-psycopg2 \
+#DOCS:    :::sh
-                        python-numpy php-phpunit-PHPUnit
+    sudo yum install -y python34-pip python34-setuptools python34-devel \
-    pip install --user --upgrade pip setuptools lettuce==0.2.18 six==1.9 \
+                        php-phpunit-PHPUnit
-                                 haversine Shapely pytidylib
+    pip3 install --user behave nose pytidylib psycopg2
     sudo pear install PHP_CodeSniffer
 
 #
@@ -77,7 +80,8 @@
 #
 # Next tune the postgresql configuration, which is located in 
 # `/var/lib/pgsql/data/postgresql.conf`. See section *Postgres Tuning* in
-# [the installation page](Installation.md) for the parameters to change.
+# [the installation page](../admin/Installation.md#postgresql-tuning)
+# for the parameters to change.
 #
 # Now start the postgresql service after updating this config file.
 
@@ -99,7 +103,7 @@
 # You need to create an alias to the website directory in your apache
 # configuration. Add a separate nominatim configuration to your webserver:
 
-#DOCS:```
+#DOCS:```sh
 sudo tee /etc/httpd/conf.d/nominatim.conf << EOFAPACHECONF
 <Directory "$USERHOME/build/website"> #DOCS:<Directory "$USERHOME/Nominatim/build/website">
   Options FollowSymLinks MultiViews
@@ -141,12 +145,10 @@ sudo sed -i 's:#.*::' /etc/httpd/conf.d/nominatim.conf #DOCS:
 #
 # Get the source code from Github and change into the source directory
 #
-if [ "x$1" == "xyes" ]; then  #DOCS:
+if [ "x$1" == "xyes" ]; then  #DOCS:    :::sh
-
     cd $USERHOME
     git clone --recursive git://github.com/openstreetmap/Nominatim.git
     cd Nominatim
-
 else                               #DOCS:
     cd $USERHOME/Nominatim         #DOCS:
 fi                                 #DOCS:
@@ -154,14 +156,14 @@ fi                                 #DOCS:
 # When installing the latest source from github, you also need to
 # download the country grid:
 
-if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:
+if [ ! -f data/country_osm_grid.sql.gz ]; then       #DOCS:    :::sh
-    wget -O data/country_osm_grid.sql.gz http://www.nominatim.org/data/country_grid.sql.gz
+    wget -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz
 fi                                 #DOCS:
 
 # The code must be built in a separate directory. Create this directory,
 # then configure and build Nominatim in there:
 
-    cd $USERHOME                   #DOCS:
+    cd $USERHOME                   #DOCS:    :::sh
     mkdir build
     cd build
     cmake $USERHOME/Nominatim
@@ -170,7 +172,7 @@ fi                                 #DOCS:
 # You need to create a minimal configuration file that tells nominatim
 # the name of your webserver user and the URL of the website:
 
-#DOCS:```
+#DOCS:```sh
 tee settings/local.php << EOF
 <?php
  @define('CONST_Database_Web_User', 'apache');
@@ -180,4 +182,4 @@ EOF
 
 
 # Nominatim is now ready to use. Continue with
-# [importing a database from OSM data](Import-and-Update.md).
+# [importing a database from OSM data](../admin/Import-and-Update.md).