prepare for 3.4.1 release

Specialphrases quotes

CMakeLists.txt

@@ -19,7 +19,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 project(nominatim)
 
 set(NOMINATIM_VERSION_MAJOR 3)
-set(NOMINATIM_VERSION_MINOR 3)
+set(NOMINATIM_VERSION_MINOR 4)
 set(NOMINATIM_VERSION_PATCH 1)
 
 set(NOMINATIM_VERSION "${NOMINATIM_VERSION_MAJOR}.${NOMINATIM_VERSION_MINOR}.${NOMINATIM_VERSION_PATCH}")
@@ -35,57 +35,60 @@ add_definitions(-DNOMINATIM_VERSION="${NOMINATIM_VERSION}")
 
 set(BUILD_TESTS off CACHE BOOL "Build test suite" FORCE)
 set(WITH_LUA off CACHE BOOL "Build with lua support" FORCE)
+set(ONLY_DOCS off CACHE BOOL "Build documentation only")
 
-if (NOT EXISTS "${CMAKE_SOURCE_DIR}/osm2pgsql/CMakeLists.txt")
-    message(FATAL_ERROR "The osm2pgsql directory is empty.\
-    Did you forget to check out Nominatim recursively?\
-    \nTry updating submodules with: git submodule update --init")
+if (NOT ONLY_DOCS)
+    if (NOT EXISTS "${CMAKE_SOURCE_DIR}/osm2pgsql/CMakeLists.txt")
+        message(FATAL_ERROR "The osm2pgsql directory is empty.\
+        Did you forget to check out Nominatim recursively?\
+        \nTry updating submodules with: git submodule update --init")
+    endif()
+    add_subdirectory(osm2pgsql)
+
+    find_package(Threads REQUIRED)
+
+    unset(PostgreSQL_TYPE_INCLUDE_DIR CACHE)
+    set(PostgreSQL_TYPE_INCLUDE_DIR "/usr/include/")
+    find_package(PostgreSQL REQUIRED)
+    include_directories(${PostgreSQL_INCLUDE_DIRS})
+    link_directories(${PostgreSQL_LIBRARY_DIRS})
+
+    find_program(PYOSMIUM pyosmium-get-changes)
+    if (NOT EXISTS "${PYOSMIUM}")
+        set(PYOSMIUM_PATH "")
+            message(WARNING "pyosmium-get-changes not found (required for updates)")
+    else()
+        set(PYOSMIUM_PATH "${PYOSMIUM}")
+        message(STATUS "Using pyosmium-get-changes at ${PYOSMIUM_PATH}")
+    endif()
+
+
+    find_program(PG_CONFIG pg_config)
+    execute_process(COMMAND ${PG_CONFIG} --pgxs
+                    OUTPUT_VARIABLE PGXS
+                    OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+    if (NOT EXISTS "${PGXS}")
+        message(FATAL_ERROR "Postgresql server package not found.")
+    endif()
+
+    find_package(ZLIB REQUIRED)
+
+    find_package(BZip2 REQUIRED)
+
+    find_package(LibXml2 REQUIRED)
+    include_directories(${LIBXML2_INCLUDE_DIR})
+
+    # Setting PHP binary variable as to command line (prevailing) or auto detect
+    if (NOT PHP_BIN)
+         find_program (PHP_BIN php)
+    endif()
+    # sanity check if PHP binary exists
+    if (NOT EXISTS ${PHP_BIN})
+        message(FATAL_ERROR "PHP binary not found. Install php or provide location with -DPHP_BIN=/path/php ")
+    endif()
+    message (STATUS "Using PHP binary " ${PHP_BIN})
 endif()
-add_subdirectory(osm2pgsql)
-
-find_package(Threads REQUIRED)
-
-unset(PostgreSQL_TYPE_INCLUDE_DIR CACHE)
-set(PostgreSQL_TYPE_INCLUDE_DIR "/usr/include/")
-find_package(PostgreSQL REQUIRED)
-include_directories(${PostgreSQL_INCLUDE_DIRS})
-link_directories(${PostgreSQL_LIBRARY_DIRS})
-
-find_program(PYOSMIUM pyosmium-get-changes)
-if (NOT EXISTS "${PYOSMIUM}")
-    set(PYOSMIUM_PATH "")
-        message(WARNING "pyosmium-get-changes not found (required for updates)")
-else()
-    set(PYOSMIUM_PATH "${PYOSMIUM}")
-    message(STATUS "Using pyosmium-get-changes at ${PYOSMIUM_PATH}")
-endif()
-
-
-find_program(PG_CONFIG pg_config)
-execute_process(COMMAND ${PG_CONFIG} --pgxs
-                OUTPUT_VARIABLE PGXS
-                OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-if (NOT EXISTS "${PGXS}")
-    message(FATAL_ERROR "Postgresql server package not found.")
-endif()
-
-find_package(ZLIB REQUIRED)
-
-find_package(BZip2 REQUIRED)
-
-find_package(LibXml2 REQUIRED)
-include_directories(${LIBXML2_INCLUDE_DIR})
-
-# Setting PHP binary variable as to command line (prevailing) or auto detect
-if (NOT PHP_BIN)
-     find_program (PHP_BIN php)
-endif()
-# sanity check if PHP binary exists
-if (NOT EXISTS ${PHP_BIN})
-    message(FATAL_ERROR "PHP binary not found. Install php or provide location with -DPHP_BIN=/path/php ")
-endif()
-message (STATUS "Using PHP binary " ${PHP_BIN})
 
 #-----------------------------------------------------------------------------
 #
@@ -143,26 +146,30 @@ endforeach()
 #
 #-----------------------------------------------------------------------------
 
-include(CTest)
+if (NOT ONLY_DOCS)
+    include(CTest)
 
-set(TEST_BDD db osm2pgsql api)
+    set(TEST_BDD db osm2pgsql api)
 
-foreach (test ${TEST_BDD})
-    add_test(NAME bdd_${test}
-             COMMAND lettuce features/${test}
-             WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests)
-    set_tests_properties(bdd_${test}
-        PROPERTIES ENVIRONMENT "NOMINATIM_DIR=${PROJECT_BINARY_DIR}")
-endforeach()
+    foreach (test ${TEST_BDD})
+        add_test(NAME bdd_${test}
+                 COMMAND lettuce features/${test}
+                 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests)
+        set_tests_properties(bdd_${test}
+            PROPERTIES ENVIRONMENT "NOMINATIM_DIR=${PROJECT_BINARY_DIR}")
+    endforeach()
 
-add_test(NAME php
-         COMMAND phpunit ./
-         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests-php)
+    add_test(NAME php
+             COMMAND phpunit ./
+             WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests-php)
+endif()
 
 #-----------------------------------------------------------------------------
 
-add_subdirectory(module)
-add_subdirectory(nominatim)
+if (NOT ONLY_DOCS)
+    add_subdirectory(module)
+    add_subdirectory(nominatim)
+endif()
 add_subdirectory(docs)
 
 #-----------------------------------------------------------------------------

ChangeLog

@@ -1,5 +1,26 @@
-3.3.1
- * security fix: fix possible SQL injection via details API
+3.4.1
+ * update osm2pgsql
+    * move deletion to copy thread (fixes deadlock in updates)
+    * fix filtering where valid address objects got dropped
+ * fix typo in import styles
+
+3.4.0
+
+ * increase required version for PostgreSQL(9.3), PostGIS(2.2) and PHP(7.0)
+ * better error reporting for out-of-memory errors
+ * exclude postcode ranges separated by colon from centre point calculation
+ * update osm2pgsql, better handling of imports without flatnode file
+ * switch to more efficient algorithm for word set computation
+ * use only boundries for country and state parts of addresses
+ * improve updates of addresses with housenumbers and interpolations
+ * remove country from place_addressline table and use country_code instead
+ * optimise indexes on search_name partition tables
+ * improve searching of attached streets for large objects like airports
+ * drop support for python 2
+ * new scripts for importing Wikidata for importance
+ * create and drop indexes concurrently to not clash with auto vacuum
+ * various documentation improvements
+
 
 3.3.0
 

data-sources/us-tiger/README.md

@@ -2,28 +2,25 @@
 
 Convert [TIGER](https://www.census.gov/geo/maps-data/data/tiger.html)/Line dataset of the US Census Bureau to SQL files which can be imported by Nominatim. The created tables in the Nominatim database are separate from OpenStreetMap tables and get queried at search time separately.
 
-The dataset gets updated once per year. Downloading is prown to be slow (can take a full day) and converting them can take hours as well.
+The dataset gets updated once per year. Downloading is prone to be slow (can take a full day) and converting them can take hours as well.
 
-Replace '2018' with the current year throughout.
+Replace '2019' with the current year throughout.
 
   1. Install the GDAL library and python bindings and the unzip tool
 
         # Ubuntu:
-        sudo apt-get install python-gdal unzip
-        # CentOS:
-        sudo yum install gdal-python unzip
+        sudo apt-get install python3-gdal unzip
 
-  2. Get the TIGER 2018 data. You will need the EDGES files
+  2. Get the TIGER 2019 data. You will need the EDGES files
      (3,233 zip files, 11GB total).
 
-         wget -r ftp://ftp2.census.gov/geo/tiger/TIGER2018/EDGES/
+         wget -r ftp://ftp2.census.gov/geo/tiger/TIGER2019/EDGES/
 
   3. Convert the data into SQL statements. Adjust the file paths in the scripts as needed
 
         cd data-sources/us-tiger
         ./convert.sh <input-path> <output-path>
-        
+
   4. Maybe: package the created files
   
-        tar -czf tiger2018-nominatim-preprocessed.tar.gz tiger
-        
+        tar -czf tiger2019-nominatim-preprocessed.tar.gz tiger

data-sources/us-tiger/tiger_address_convert.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # Tiger road data to OSM conversion script
 # Creates Karlsruhe-style address ways beside the main way
 # based on the Massachusetts GIS script by christopher schmidt
@@ -164,7 +164,7 @@ def parse_shp_for_geom_and_tags( filename ):
         if (statefp != None) and (countyfp != None):
             county_name = county_fips_data.get(statefp + '' + countyfp)
             if county_name:
-                tags["tiger:county"] = county_name.encode("utf-8")
+                tags["tiger:county"] = county_name
 
         # tlid = poFeature.GetField("TLID")
         # if tlid != None:

data-sources/wikipedia-wikidata/README.md

@@ -0,0 +1,58 @@
+## Add Wikipedia and Wikidata to Nominatim
+
+OSM contributors frequently tag items with links to Wikipedia and Wikidata. Nominatim can use the page ranking of Wikipedia pages to help indicate the relative importance of osm features. This is done by calculating an importance score between 0 and 1 based on the number of inlinks to an article for a location. If two places have the same name and one is more important than the other, the wikipedia score often points to the correct place. 
+
+These scripts extract and prepare both Wikipedia page rank and Wikidata links for use in Nominatim.  
+
+#### Create a new postgres DB for Processing
+
+Due to the size of initial and intermediate tables, processing can be done in an external database:
+```
+CREATE DATABASE wikiprocessingdb;
+```
+---
+Wikipedia
+---  
+
+Processing these data requires a large amount of disk space (~1TB) and considerable time (>24 hours).
+
+#### Import & Process Wikipedia tables
+
+This step downloads and converts [Wikipedia](https://dumps.wikimedia.org/) page data SQL dumps to postgreSQL files which can be imported and processed with pagelink information from Wikipedia language sites to calculate importance scores.
+
+- The script will processes data from whatever set of Wikipedia languages are specified in the initial languages array
+
+- Note that processing the top 40 Wikipedia languages can take over a day, and will add nearly 1TB to the processing database. The final output tables will be approximately 11GB and 2GB in size
+
+To download, convert, and import the data, then process summary statistics and compute importance scores, run:
+```
+./wikipedia_import.sh
+```
+---
+Wikidata
+---
+
+This script downloads and processes Wikidata to enrich the previously created Wikipedia tables for use in Nominatim.
+
+#### Import & Process Wikidata
+
+This step downloads and converts [Wikidata](https://dumps.wikimedia.org/wikidatawiki/) page data SQL dumps to postgreSQL files which can be processed and imported into Nominatim database. Also utilizes Wikidata Query Service API to discover and include place types.
+
+- Script presumes that the user has already processed Wikipedia tables as specified above
+
+- Script requires wikidata_place_types.txt and wikidata_place_type_levles.csv
+
+- script requires the [jq json parser](https://stedolan.github.io/jq/)
+
+- Script processes data from whatever set of Wikipedia languages are specified in the initial languages array
+
+- Script queries Wikidata Query Service API and imports all instances of place types listed in wikidata_place_types.txt
+
+- Script updates wikipedia_articles table with extracted wikidata 
+
+By including Wikidata in the wikipedia_articles table, new connections can be made on the fly from the Nominatim placex table to wikipedia_article importance scores. 
+
+To download, convert, and import the data, then process required items, run:
+``` 
+./wikidata_import.sh
+```

data-sources/wikipedia-wikidata/import_wikidata.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+psqlcmd() {
+     psql wikiprocessingdb
+}
+
+mysql2pgsqlcmd() {
+     ./mysql2pgsql.perl /dev/stdin /dev/stdout
+}
+
+
+# list the languages to process (refer to List of Wikipedias here: https://en.wikipedia.org/wiki/List_of_Wikipedias)
+
+language=( "ar" "bg" "ca" "cs" "da" "de" "en" "es" "eo" "eu" "fa" "fr" "ko" "hi" "hr" "id" "it" "he" "lt" "hu" "ms" "nl" "ja" "no" "pl" "pt" "kk" "ro" "ru" "sk" "sl" "sr" "fi" "sv" "tr" "uk" "vi" "vo" "war" "zh" )
+
+
+# get a few wikidata dump tables
+
+wget https://dumps.wikimedia.org/wikidatawiki/latest/wikidatawiki-latest-geo_tags.sql.gz
+wget https://dumps.wikimedia.org/wikidatawiki/latest/wikidatawiki-latest-page.sql.gz
+wget https://dumps.wikimedia.org/wikidatawiki/latest/wikidatawiki-latest-wb_items_per_site.sql.gz
+
+
+# import wikidata tables
+
+gzip -dc wikidatawiki-latest-geo_tags.sql.gz | mysql2pgsqlcmd | psqlcmd
+gzip -dc wikidatawiki-latest-page.sql.gz | mysql2pgsqlcmd | psqlcmd
+gzip -dc wikidatawiki-latest-wb_items_per_site.sql.gz | mysql2pgsqlcmd | psqlcmd
+
+
+# get wikidata places from wikidata query API
+
+while read F  ; do
+    wget "https://query.wikidata.org/bigdata/namespace/wdq/sparql?format=json&query=SELECT ?item WHERE{?item wdt:P31*/wdt:P279*wd:$F;}" -O $F.json
+    jq -r '.results | .[] | .[] | [.item.value] | @csv' $F.json >> $F.txt
+    awk -v qid=$F '{print $0 ","qid}' $F.txt | sed -e 's!"http://www.wikidata.org/entity/!!' | sed 's/"//g' >> $F.csv
+    cat $F.csv >> wikidata_place_dump.csv
+    rm $F.json $F.txt $F.csv
+done < wikidata_place_types.txt
+
+
+# import wikidata places
+
+echo "CREATE TABLE wikidata_place_dump (item text, instance_of text);"  | psqlcmd
+echo "COPY wikidata_place_dump (item, instance_of) FROM  '/srv/nominatim/Nominatim/data-sources/wikipedia-wikidata/wikidata_place_dump.csv' DELIMITER ',' CSV;"  | psqlcmd
+
+echo "CREATE TABLE wikidata_place_type_levels (place_type text, level integer);" | psqlcmd
+echo "COPY wikidata_place_type_levels (place_type, level) FROM '/srv/nominatim/Nominatim/data-sources/wikipedia-wikidata/wikidata_place_type_levels.csv' DELIMITER ',' CSV HEADER;" | psqlcmd
+
+
+# create derived tables
+
+echo "CREATE TABLE geo_earth_primary AS SELECT gt_page_id, gt_lat, gt_lon FROM geo_tags WHERE gt_globe = 'earth' AND gt_primary = 1 AND NOT( gt_lat < -90 OR gt_lat > 90 OR gt_lon < -180 OR gt_lon > 180 OR gt_lat=0 OR gt_lon=0) ;" | psqlcmd
+echo "CREATE TABLE geo_earth_wikidata AS SELECT DISTINCT geo_earth_primary.gt_page_id, geo_earth_primary.gt_lat, geo_earth_primary.gt_lon, page.page_title, page.page_namespace FROM geo_earth_primary LEFT OUTER JOIN page ON (geo_earth_primary.gt_page_id = page.page_id) ORDER BY geo_earth_primary.gt_page_id;" | psqlcmd
+
+echo "ALTER TABLE wikidata_place_dump ADD COLUMN ont_level integer, ADD COLUMN lat numeric(11,8), ADD COLUMN lon numeric(11,8);" | psqlcmd
+echo "UPDATE wikidata_place_dump SET ont_level = wikidata_place_type_levels.level FROM wikidata_place_type_levels WHERE wikidata_place_dump.instance_of = wikidata_place_type_levels.place_type;" | psqlcmd
+
+echo "CREATE TABLE wikidata_places AS SELECT DISTINCT ON (item) item, instance_of, MAX(ont_level) AS ont_level, lat, lon FROM wikidata_place_dump GROUP BY item, instance_of, ont_level, lat, lon ORDER BY item;" | psqlcmd
+echo "UPDATE wikidata_places SET lat = geo_earth_wikidata.gt_lat, lon = geo_earth_wikidata.gt_lon FROM geo_earth_wikidata WHERE wikidata_places.item = geo_earth_wikidata.page_title" | psqlcmd
+
+
+# process language pages 
+
+echo "CREATE TABLE wikidata_pages (item text, instance_of text, lat numeric(11,8), lon numeric(11,8), ips_site_page text, language text );" | psqlcmd
+
+for i in "${language[@]}"
+do
+   echo "CREATE TABLE wikidata_${i}_pages as select wikidata_places.item, wikidata_places.instance_of, wikidata_places.lat, wikidata_places.lon, wb_items_per_site.ips_site_page FROM wikidata_places LEFT JOIN wb_items_per_site ON (CAST (( LTRIM(wikidata_places.item, 'Q')) AS INTEGER) = wb_items_per_site.ips_item_id) WHERE ips_site_id = '${i}wiki' AND LEFT(wikidata_places.item,1) = 'Q' order by wikidata_places.item;" | psqlcmd
+   echo "ALTER TABLE wikidata_${i}_pages ADD COLUMN language text;" | psqlcmd
+   echo "UPDATE wikidata_${i}_pages SET language = '${i}';" | psqlcmd
+   echo "INSERT INTO wikidata_pages SELECT item, instance_of, lat, lon, ips_site_page, language FROM wikidata_${i}_pages;" | psqlcmd
+done
+
+echo "ALTER TABLE wikidata_pages ADD COLUMN wp_page_title text;" | psqlcmd
+echo "UPDATE wikidata_pages SET wp_page_title = REPLACE(ips_site_page, ' ', '_');" | psqlcmd
+echo "ALTER TABLE wikidata_pages DROP COLUMN ips_site_page;" | psqlcmd
+
+
+# add wikidata to wikipedia_article table
+
+echo "UPDATE wikipedia_article SET lat = wikidata_pages.lat, lon = wikidata_pages.lon, wd_page_title = wikidata_pages.item, instance_of = wikidata_pages.instance_of FROM wikidata_pages WHERE wikipedia_article.language = wikidata_pages.language AND wikipedia_article.title  = wikidata_pages.wp_page_title;" | psqlcmd
+echo "CREATE TABLE wikipedia_article_slim AS SELECT * FROM wikipedia_article WHERE wikidata_id IS NOT NULL;" | psqlcmd
+echo "ALTER TABLE wikipedia_article RENAME TO wikipedia_article_full;" | psqlcmd
+echo "ALTER TABLE wikipedia_article_slim RENAME TO wikipedia_article;" | psqlcmd
+
+
+# clean up intermediate tables
+
+echo "DROP TABLE wikidata_place_dump;" | psqlcmd
+echo "DROP TABLE geo_earth_primary;" | psqlcmd
+for i in "${language[@]}"
+do
+    echo "DROP TABLE wikidata_${i}_pages;" | psqlcmd
+done

data-sources/wikipedia-wikidata/import_wikipedia.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+psqlcmd() {
+     psql wikiprocessingdb
+}
+
+mysql2pgsqlcmd() {
+     ./mysql2pgsql.perl /dev/stdin /dev/stdout
+}
+
+
+# list the languages to process (refer to List of Wikipedias here: https://en.wikipedia.org/wiki/List_of_Wikipedias)
+
+language=( "ar" "bg" "ca" "cs" "da" "de" "en" "es" "eo" "eu" "fa" "fr" "ko" "hi" "hr" "id" "it" "he" "lt" "hu" "ms" "nl" "ja" "no" "pl" "pt" "kk" "ro" "ru" "sk" "sl" "sr" "fi" "sv" "tr" "uk" "vi" "vo" "war" "zh" )
+
+
+# create wikipedia calculation tables
+
+echo "CREATE TABLE linkcounts (language text, title text, count integer, sumcount integer, lat double precision, lon double precision);"  | psqlcmd
+echo "CREATE TABLE wikipedia_article (language text NOT NULL, title text NOT NULL, langcount integer, othercount integer, totalcount integer, lat double precision, lon double precision, importance double precision, title_en text, osm_type character(1), osm_id bigint );" | psqlcmd
+echo "CREATE TABLE wikipedia_redirect (language text, from_title text, to_title text );"  | psqlcmd
+
+
+# download individual wikipedia language tables
+
+for i in "${language[@]}"
+do
+    wget https://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-page.sql.gz
+    wget https://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-pagelinks.sql.gz
+    wget https://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-langlinks.sql.gz
+    wget https://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-redirect.sql.gz
+done
+
+
+# import individual wikipedia language tables
+
+for i in "${language[@]}"
+do
+    gzip -dc ${i}wiki-latest-pagelinks.sql.gz | sed "s/\`pagelinks\`/\`${i}pagelinks\`/g" | mysql2pgsqlcmd | psqlcmd
+    gzip -dc ${i}wiki-latest-page.sql.gz | sed "s/\`page\`/\`${i}page\`/g" | mysql2pgsqlcmd | psqlcmd
+    gzip -dc ${i}wiki-latest-langlinks.sql.gz | sed "s/\`langlinks\`/\`${i}langlinks\`/g" | mysql2pgsqlcmd | psqlcmd
+    gzip -dc ${i}wiki-latest-redirect.sql.gz | sed "s/\`redirect\`/\`${i}redirect\`/g" | mysql2pgsqlcmd | psqlcmd
+done
+
+
+# process language tables and associated pagelink counts
+
+for i in "${language[@]}"
+do
+    echo "create table ${i}pagelinkcount as select pl_title as title,count(*) as count from ${i}pagelinks where pl_namespace = 0 group by pl_title;" | psqlcmd
+    echo "insert into linkcounts select '${i}',pl_title,count(*) from ${i}pagelinks where pl_namespace = 0 group by pl_title;" | psqlcmd
+    echo "insert into wikipedia_redirect select '${i}',page_title,rd_title from ${i}redirect join ${i}page on (rd_from = page_id) where page_namespace = 0 and rd_namespace = 0;" | psqlcmd
+    echo "alter table ${i}pagelinkcount add column othercount integer;" | psqlcmd
+    echo "update ${i}pagelinkcount set othercount = 0;" | psqlcmd
+    for j in "${language[@]}"
+    do
+        echo "update ${i}pagelinkcount set othercount = ${i}pagelinkcount.othercount + x.count from (select page_title as title,count from ${i}langlinks join ${i}page on (ll_from = page_id) join ${j}pagelinkcount on (ll_lang = '${j}' and ll_title = title)) as x where x.title = ${i}pagelinkcount.title;" | psqlcmd
+    done
+    echo "insert into wikipedia_article select '${i}', title, count, othercount, count+othercount from ${i}pagelinkcount;" | psqlcmd
+done
+
+
+# calculate importance score for each wikipedia page
+
+echo "update wikipedia_article set importance = log(totalcount)/log((select max(totalcount) from wikipedia_article))" | psqlcmd
+
+
+# clean up intermediate tables to conserve space
+
+for i in "${language[@]}"
+do
+    echo "DROP TABLE ${i}pagelinks;" | psqlcmd
+    echo "DROP TABLE ${i}page;" | psqlcmd
+    echo "DROP TABLE ${i}langlinks;" | psqlcmd
+    echo "DROP TABLE ${i}redirect;" | psqlcmd
+    echo "DROP TABLE ${i}pagelinkcount;" | psqlcmd
+done

data-sources/wikipedia-wikidata/mysql2pgsql.perl

@@ -341,7 +341,7 @@ if (/(create\s+table\s+)([-_\w]+)\s/i) { #  example: CREATE TABLE `english_engli
         #  in the foreign-key case it will only remove the foreign-key constraint, not the other table entirely.)
         #  (source: 8.1.3 docs, section "drop table")
         warn "table $table will be dropped CASCADE\n";
-        $pre_create_sql .= "DROP TABLE $table CASCADE\\g\n";    # custom dumps may be missing the 'dump' commands
+        $pre_create_sql .= "DROP TABLE $table CASCADE;\n";    # custom dumps may be missing the 'dump' commands
     }
 
     s/(create\s+table\s+)([-_\w]+)\s/$1 $table /i;
@@ -367,6 +367,7 @@ if ($create_sql ne "") {         # we are inside create table statement so lets
         s/INSERT METHOD[=\s+][^;\s]+//i;
         s/PASSWORD=[^;\s]+//i;
         s/ROW_FORMAT=(?:DEFAULT|DYNAMIC|FIXED|COMPRESSED|REDUNDANT|COMPACT)+//i;
+        s/KEY_BLOCK_SIZE=8//i;
         s/DELAY KEY WRITE=[^;\s]+//i;
         s/INDEX DIRECTORY[=\s+][^;\s]+//i;
         s/DATA DIRECTORY=[^;\s]+//i;
@@ -389,6 +390,7 @@ if ($create_sql ne "") {         # we are inside create table statement so lets
         s/DEFAULT CHARSET=[^;\s]+//i; #  my mysql version is 4.1.11
         s/ENGINE\s*=\s*[^;\s]+//i;   #  my mysql version is 4.1.11
         s/ROW_FORMAT=[^;\s]+//i;   #  my mysql version is 5.0.22
+        s/KEY_BLOCK_SIZE=8//i; 
         s/MIN_ROWS=[^;\s]+//i;
         s/MAX_ROWS=[^;\s]+//i;
         s/AVG_ROW_LENGTH=[^;\s]+//i;

data-sources/wikipedia-wikidata/wikidata_place_type_levels.csv

@@ -0,0 +1,199 @@
+place_type,level
+Q9842,4
+Q9430,3
+Q928830,4
+Q9259,1
+Q91028,5
+Q8514,2
+Q8502,2
+Q83405,3
+Q82794,2
+Q820477,1
+Q811979,1
+Q8072,2
+Q79007,2
+Q786014,3
+Q75848,2
+Q75520,2
+Q728937,4
+Q7275,2
+Q719456,3
+Q7075,3
+Q697295,4
+Q6852233,2
+Q682943,3
+Q665487,5
+Q655686,3
+Q643589,5
+Q641226,2
+Q631305,2
+Q6256,2
+Q6023295,2
+Q5773747,5
+Q56061,1
+Q55659167,4
+Q55488,4
+Q55465477,3
+Q54050,2
+Q532,3
+Q53060,2
+Q52177058,4
+Q515716,5
+Q5153984,4
+Q515,3
+Q5144960,5
+Q5119,4
+Q5119,4
+Q5107,2
+Q5084,4
+Q5031071,4
+Q5003624,2
+Q4989906,1
+Q4976993,3
+Q486972,1
+Q486972,2
+Q483110,3
+Q4830453,4
+Q47521,3
+Q473972,1
+Q46831,2
+Q46614560,5
+Q44782,3
+Q44613,4
+Q44539,4
+Q44494,2
+Q44377,2
+Q4421,2
+Q43501,2
+Q4286337,3
+Q42523,3
+Q41176,2
+Q40357,3
+Q4022,4
+Q40080,2
+Q39816,2
+Q39715,3
+Q39614,1
+Q3957,3
+Q3947,4
+Q3914,3
+Q38723,2
+Q38720,3
+Q3623867,5
+Q35666,2
+Q355304,3
+Q35509,2
+Q35112127,3
+Q34985575,4
+Q34876,5
+Q34763,2
+Q34627,4
+Q3455524,3
+Q34442,4
+Q33837,2
+Q33506,3
+Q32815,4
+Q3257686,2
+Q3240715,2
+Q3191695,5
+Q3153117,2
+Q30198,2
+Q30139652,3
+Q294422,3
+Q2870166,3
+Q27686,3
+Q274153,3
+Q271669,1
+Q2659904,2
+Q24529780,2
+Q24354,3
+Q2354973,4
+Q23442,2
+Q23413,3
+Q23397,3
+Q2327515,4
+Q2311958,5
+Q22927291,6
+Q22698,1
+Q2175765,4
+Q205495,4
+Q204832,3
+Q2042028,2
+Q202216,6
+Q1970725,3
+Q194203,5
+Q194195,2
+Q190429,2
+Q185187,3
+Q185113,2
+Q183366,2
+Q1799794,1
+Q1788454,4
+Q1785071,3
+Q1777138,3
+Q177634,2
+Q177380,2
+Q174814,4
+Q174782,2
+Q17350442,2
+Q17343829,3
+Q17334923,0
+Q17018380,3
+Q16970,4
+Q16917,3
+Q16831714,4
+Q165,3
+Q160742,4
+Q159719,3
+Q159334,4
+Q15640612,5
+Q15324,2
+Q15284,5
+Q15243209,6
+Q152081,1
+Q15195406,4
+Q1500350,5
+Q149621,5
+Q14757767,4
+Q14350,3
+Q1410668,3
+Q1394476,3
+Q1377575,2
+Q1353183,3
+Q134447,4
+Q133215,3
+Q133056,2
+Q13221722,3
+Q13220204,2
+Q1311958,4
+Q1303167,3
+Q130003,3
+Q12518,2
+Q12516,3
+Q1248784,3
+Q123705,3
+Q12323,3
+Q12284,4
+Q12280,4
+Q121359,2
+Q1210950,2
+Q11755880,3
+Q11707,3
+Q11315,3
+Q11303,3
+Q1115575,4
+Q1107656,1
+Q10864048,1
+Q1076486,2
+Q105731,3
+Q105190,3
+Q1048525,3
+Q102496,5
+Q28872924,1
+Q15617994,1
+Q159313,2
+Q24398318,3
+Q327333,2
+Q43229,1
+Q860861,1
+Q4989906,1

data-sources/wikipedia-wikidata/wikidata_place_types.txt

@@ -0,0 +1,195 @@
+Q9842
+Q9430
+Q928830
+Q9259
+Q91028
+Q8514
+Q8502
+Q83405
+Q82794
+Q820477
+Q811979
+Q8072
+Q79007
+Q786014
+Q75848
+Q75520
+Q728937
+Q7275
+Q719456
+Q7075
+Q697295
+Q6852233
+Q682943
+Q665487
+Q655686
+Q643589
+Q641226
+Q631305
+Q6256
+Q6023295
+Q5773747
+Q56061
+Q55659167
+Q55488
+Q55465477
+Q54050
+Q532
+Q53060
+Q52177058
+Q515716
+Q5153984
+Q515
+Q5144960
+Q5119
+Q5107
+Q5084
+Q5031071
+Q5003624
+Q4989906
+Q4976993
+Q486972
+Q483110
+Q4830453
+Q47521
+Q473972
+Q46831
+Q46614560
+Q44782
+Q44613
+Q44539
+Q44494
+Q44377
+Q4421
+Q43501
+Q4286337
+Q42523
+Q41176
+Q40357
+Q4022
+Q40080
+Q39816
+Q39715
+Q39614
+Q3957
+Q3947
+Q3914
+Q38723
+Q38720
+Q3623867
+Q35666
+Q355304
+Q35509
+Q35112127
+Q34985575
+Q34876
+Q34763
+Q34627
+Q3455524
+Q34442
+Q33837
+Q33506
+Q32815
+Q3257686
+Q3240715
+Q3191695
+Q3153117
+Q30198
+Q30139652
+Q294422
+Q2870166
+Q27686
+Q274153
+Q271669
+Q2659904
+Q24529780
+Q24354
+Q2354973
+Q23442
+Q23413
+Q23397
+Q2327515
+Q2311958
+Q22927291
+Q22698
+Q2175765
+Q205495
+Q204832
+Q2042028
+Q202216
+Q1970725
+Q194203
+Q194195
+Q190429
+Q185187
+Q185113
+Q183366
+Q1799794
+Q1788454
+Q1785071
+Q1777138
+Q177634
+Q177380
+Q174814
+Q174782
+Q17350442
+Q17343829
+Q17334923
+Q17018380
+Q16970
+Q16917
+Q16831714
+Q165
+Q160742
+Q159719
+Q159334
+Q15640612
+Q15324
+Q15284
+Q15243209
+Q152081
+Q15195406
+Q1500350
+Q149621
+Q14757767
+Q14350
+Q1410668
+Q1394476
+Q1377575
+Q1353183
+Q134447
+Q133215
+Q133056
+Q13221722
+Q13220204
+Q1311958
+Q1303167
+Q130003
+Q12518
+Q12516
+Q1248784
+Q123705
+Q12323
+Q12284
+Q12280
+Q121359
+Q1210950
+Q11755880
+Q11707
+Q11315
+Q11303
+Q1115575
+Q1107656
+Q10864048
+Q1076486
+Q105731
+Q105190
+Q1048525
+Q102496
+Q28872924
+Q15617994
+Q159313
+Q24398318
+Q327333
+Q43229
+Q860861

data-sources/wikipedia-wikidata/wikidata_places.md

@@ -0,0 +1,200 @@
+
+## Wikidata place types and related OSM Tags
+
+Wikidata does not have any official ontologies, however the [DBpedia project](https://wiki.dbpedia.org/) has created an [ontology](https://wiki.dbpedia.org/services-resources/ontology) that covered [place types](http://mappings.dbpedia.org/server/ontology/classes/#Place). The table below used the DBpedia place ontology as a starting point, and is provided as a cross-reference to the relevant OSM tags.
+
+The Wikidata place types listed in the table below can be used in conjunction with the [Wikidata Query Service](https://query.wikidata.org/) to retrieve instances of those place types from the Wikidata knowledgebase. 
+
+```
+SELECT ?item ?lat ?lon
+WHERE {
+  ?item wdt:P31*/wdt:P279*wd:Q9430; wdt:P625 ?pt.
+  ?item p:P625?loc.
+  ?loc psv:P625?cnode.
+  ?cnode wikibase:geoLatitude?lat.
+  ?cnode wikibase:geoLongitude?lon.
+}
+```
+
+An example json return for all instances of the Wikidata item "Q9430" (Ocean) can be seen at [json](https://query.wikidata.org/bigdata/namespace/wdq/sparql?format=json&query=SELECT?item?lat?lon%20WHERE{?item%20wdt:P31*/wdt:P279*wd:Q9430;wdt:P625?pt.?item%20p:P625?loc.?loc%20psv:P625?cnode.?cnode%20wikibase:geoLatitude?lat.?cnode%20wikibase:geoLongitude?lon.})
+
+**NOTE** the OSM tags listed are those listed in the wikidata entries, and not all the possible matches for tags within OSM.
+
+
+   title   |             concept                   |       OSM Tag     | 
+-----------|---------------------------------------|------------------|
+[Q17334923](https://www.wikidata.org/entity/Q17334923)  | Location | | 
+[Q811979](https://www.wikidata.org/entity/Q811979)           | Architectural Structure | | 
+[Q194195](https://www.wikidata.org/entity/Q194195)   | Amusement park | 
+[Q204832](https://www.wikidata.org/entity/Q204832)   | Roller coaster | [attraction=roller_coaster](https://wiki.openstreetmap.org/wiki/Tag:attraction=roller_coaster) | 
+[Q2870166](https://www.wikidata.org/entity/Q2870166)   | Water ride | |
+[Q641226](https://www.wikidata.org/entity/Q641226)    | Arena | [amenity=events_centre](https://wiki.openstreetmap.org/wiki/Tag:amenity=events_centre) | 
+[Q41176](https://www.wikidata.org/entity/Q41176)     | Building | [building=yes](https://wiki.openstreetmap.org/wiki/Key:building) |
+[Q1303167](https://www.wikidata.org/entity/Q1303167)   | Barn | [building=barn](https://wiki.openstreetmap.org/wiki/Tag:building=barn) |
+[Q655686](https://www.wikidata.org/entity/Q655686)   | Commercial building | [building=commercial](https://wiki.openstreetmap.org/wiki/Tag:building=commercial) | 
+[Q4830453](https://www.wikidata.org/entity/Q4830453)   | Business | |
+[Q7075](https://www.wikidata.org/entity/Q7075)     | Library | [amenity=library](https://wiki.openstreetmap.org/wiki/Tag:amenity=library) |
+[Q133215](https://www.wikidata.org/entity/Q133215)   | Casino | [amenity=casino](https://wiki.openstreetmap.org/wiki/Tag:amenity=casino) | 
+[Q23413](https://www.wikidata.org/entity/Q23413)     | Castle | [historic=castle](https://wiki.openstreetmap.org/wiki/Tag:historic=castle) |
+[Q83405](https://www.wikidata.org/entity/Q83405)     | Factory | | 
+[Q53060](https://www.wikidata.org/entity/Q53060)     | Gate  | [barrier=gate](https://wiki.openstreetmap.org/wiki/Tag:barrier=gate) |cnode%20wikibase:geoLatitude?lat.?cnode%20wikibase:geoLongitude?lon.})
+[Q11755880](https://www.wikidata.org/entity/Q11755880)           | Residential Building  | [building=residential](https://wiki.openstreetmap.org/wiki/Tag:building=residential) | 
+[Q3947](https://www.wikidata.org/entity/Q3947)      | House  | [building=house](https://wiki.openstreetmap.org/wiki/Tag:building=house) |
+[Q35112127](https://www.wikidata.org/entity/Q35112127)           | Historic Building  | |
+[Q5773747](https://www.wikidata.org/entity/Q5773747)   | Historic house  | | 
+[Q38723](https://www.wikidata.org/entity/Q38723)           | Higher Education Institution  | 
+[Q3914](https://www.wikidata.org/entity/Q3914)      | School  | [amenity=school](https://wiki.openstreetmap.org/wiki/Tag:amenity=school) | 
+[Q9842](https://www.wikidata.org/entity/Q9842)      | Primary school  | | 
+[Q159334](https://www.wikidata.org/entity/Q159334)    | Secondary school  | | 
+[Q16917](https://www.wikidata.org/entity/Q16917)     | Hospital  | [amenity=hospital](https://wiki.openstreetmap.org/wiki/Tag:amenity=hospital), [healthcare=hospital](https://wiki.openstreetmap.org/wiki/Tag:healthcare=hospital), [building=hospital](https://wiki.openstreetmap.org/wiki/Tag:building=hospital) |
+[Q27686](https://www.wikidata.org/entity/Q27686)     | Hotel  | [tourism=hotel](https://wiki.openstreetmap.org/wiki/Tag:tourism=hotel), [building=hotel](https://wiki.openstreetmap.org/wiki/Tag:building=hotel) |
+[Q33506](https://www.wikidata.org/entity/Q33506)     | Museum  | [tourism=museum](https://wiki.openstreetmap.org/wiki/Tag:tourism=museum) |
+[Q40357](https://www.wikidata.org/entity/Q40357)     | Prison  | [amenity=prison](https://wiki.openstreetmap.org/wiki/Tag:amenity=prison) |
+[Q24398318](https://www.wikidata.org/entity/Q24398318)           | Religious Building  | |
+[Q160742](https://www.wikidata.org/entity/Q160742)    | Abbey  | |
+[Q16970](https://www.wikidata.org/entity/Q16970)     | Church (building)  | [building=church](https://wiki.openstreetmap.org/wiki/Tag:building=church) |
+[Q44613](https://www.wikidata.org/entity/Q44613)     | Monastery  | [amenity=monastery](https://wiki.openstreetmap.org/wiki/Tag:amenity=monastery) | 
+[Q32815](https://www.wikidata.org/entity/Q32815)     | Mosque  | [building=mosque](https://wiki.openstreetmap.org/wiki/Tag:building=mosque) | 
+[Q697295](https://www.wikidata.org/entity/Q697295)    | Shrine  | [building=shrine](https://wiki.openstreetmap.org/wiki/Tag:building=shrine) |
+[Q34627](https://www.wikidata.org/entity/Q34627)     | Synagogue  | [building=synagogue](https://wiki.openstreetmap.org/wiki/Tag:building=synagogue) |
+[Q44539](https://www.wikidata.org/entity/Q44539)     | Temple  | [building=temple](https://wiki.openstreetmap.org/wiki/Tag:building=temple) | 
+[Q11707](https://www.wikidata.org/entity/Q11707)     | Restaurant  | [amenity=restaurant](https://wiki.openstreetmap.org/wiki/Tag:amenity=restaurant) |
+[Q11315](https://www.wikidata.org/entity/Q11315)     | Shopping mall  | [shop=mall](https://wiki.openstreetmap.org/wiki/Tag:shop=mall), [shop=shopping_centre](https://wiki.openstreetmap.org/wiki/Tag:shop=shopping_centre) | 
+[Q11303](https://www.wikidata.org/entity/Q11303)     | Skyscraper  | |
+[Q17350442](https://www.wikidata.org/entity/Q17350442)           | Venue  | |
+[Q41253](https://www.wikidata.org/entity/Q41253)           | Movie Theater  | [amenity=cinema](https://wiki.openstreetmap.org/wiki/Tag:amenity=cinema) | 
+[Q483110](https://www.wikidata.org/entity/Q483110)    | Stadium  | [leisure=stadium](https://wiki.openstreetmap.org/wiki/Tag:leisure=stadium), [building=stadium](https://wiki.openstreetmap.org/wiki/Tag:building=stadium) |
+[Q24354](https://www.wikidata.org/entity/Q24354)     | Theater (structure)  | [amenity=theatre](https://wiki.openstreetmap.org/wiki/Tag:amenity=theatre) |
+[Q121359](https://www.wikidata.org/entity/Q121359)    | Infrastructure  | |
+[Q1248784](https://www.wikidata.org/entity/Q1248784)   | Airport  | |
+[Q12323](https://www.wikidata.org/entity/Q12323)     | Dam  | [waterway=dam](https://wiki.openstreetmap.org/wiki/Tag:waterway=dam) |
+[Q1353183](https://www.wikidata.org/entity/Q1353183)   | Launch pad  | | 
+[Q105190](https://www.wikidata.org/entity/Q105190)   | Levee  | [man_made=dyke](https://wiki.openstreetmap.org/wiki/Tag:man_made=dyke) |
+[Q105731](https://www.wikidata.org/entity/Q105731)    | Lock (water navigation)   | [lock=yes](https://wiki.openstreetmap.org/wiki/Key:lock) |
+[Q44782](https://www.wikidata.org/entity/Q44782)     | Port  | |
+[Q159719](https://www.wikidata.org/entity/Q159719)    | Power station  | [power=plant](https://wiki.openstreetmap.org/wiki/Tag:power=plant) |
+[Q174814](https://www.wikidata.org/entity/Q174814)    | Electrical substation   |  |
+[Q134447](https://www.wikidata.org/entity/Q134447)    | Nuclear power plant  | [plant:source=nuclear](https://wiki.openstreetmap.org/wiki/Tag:plant:source=nuclear) |
+[Q786014](https://www.wikidata.org/entity/Q786014)   | Rest area  | [highway=rest_area](https://wiki.openstreetmap.org/wiki/Tag:highway=rest_area), [highway=services](https://wiki.openstreetmap.org/wiki/Tag:highway=services) |
+[Q12280](https://www.wikidata.org/entity/Q12280)     | Bridge  | [bridge=* ](https://wiki.openstreetmap.org/wiki/Key:bridge), [man_made=bridge](https://wiki.openstreetmap.org/wiki/Tag:man_made=bridge) |
+[Q728937](https://www.wikidata.org/entity/Q728937)           | Railroad Line  | [railway=rail](https://wiki.openstreetmap.org/wiki/Tag:railway=rail) | 
+[Q1311958](https://www.wikidata.org/entity/Q1311958)           | Railway Tunnel  | | 
+[Q34442](https://www.wikidata.org/entity/Q34442)     | Road  | [highway=* ](https://wiki.openstreetmap.org/wiki/Key:highway), [route=road](https://wiki.openstreetmap.org/wiki/Tag:route=road) |
+[Q1788454](https://www.wikidata.org/entity/Q1788454)   | Road junction  |  | 
+[Q44377](https://www.wikidata.org/entity/Q44377)     | Tunnel  | [tunnel=* ](https://wiki.openstreetmap.org/wiki/Key:tunnel) |
+[Q5031071](https://www.wikidata.org/entity/Q5031071)  | Canal tunnel  | |
+[Q719456](https://www.wikidata.org/entity/Q719456)    | Station  | [public_transport=station](https://wiki.openstreetmap.org/wiki/Tag:public_transport=station) |
+[Q205495](https://www.wikidata.org/entity/Q205495)    | Filling station  | [amenity=fuel](https://wiki.openstreetmap.org/wiki/Tag:amenity=fuel) |
+[Q928830](https://www.wikidata.org/entity/Q928830)    | Metro station  | [station=subway](https://wiki.openstreetmap.org/wiki/Tag:station=subway) |
+[Q55488](https://www.wikidata.org/entity/Q55488)     | Train station  | [railway=station](https://wiki.openstreetmap.org/wiki/Tag:railway=station) |
+[Q2175765](https://www.wikidata.org/entity/Q2175765)   | Tram stop  | [railway=tram_stop](https://wiki.openstreetmap.org/wiki/Tag:railway=tram_stop), [public_transport=stop_position](https://wiki.openstreetmap.org/wiki/Tag:public_transport=stop_position) |
+[Q6852233](https://www.wikidata.org/entity/Q6852233)   | Military building  | |
+[Q44494](https://www.wikidata.org/entity/Q44494)     | Mill (grinding)  | |
+[Q185187](https://www.wikidata.org/entity/Q185187)    | Watermill  | [man_made=watermill](https://wiki.openstreetmap.org/wiki/Tag:man_made=watermill) |
+[Q38720](https://www.wikidata.org/entity/Q38720)     | Windmill  | [man_made=windmill](https://wiki.openstreetmap.org/wiki/Tag:man_made=windmill) | 
+[Q4989906](https://www.wikidata.org/entity/Q4989906)   | Monument  | [historic=monument](https://wiki.openstreetmap.org/wiki/Tag:historic=monument) |
+[Q5003624](https://www.wikidata.org/entity/Q5003624)   | Memorial  | [historic=memorial](https://wiki.openstreetmap.org/wiki/Tag:historic=memorial) |
+[Q271669](https://www.wikidata.org/entity/Q271669)   | Landform  | |
+[Q190429](https://www.wikidata.org/entity/Q190429)    | Depression (geology)  | |
+[Q17018380](https://www.wikidata.org/entity/Q17018380)  | Bight (geography)  | | 
+[Q54050](https://www.wikidata.org/entity/Q54050)     | Hill  | |
+[Q1210950](https://www.wikidata.org/entity/Q1210950)   | Channel (geography)  | |
+[Q23442](https://www.wikidata.org/entity/Q23442)    | Island  | [place=island](https://wiki.openstreetmap.org/wiki/Tag:place=island) | 
+[Q42523](https://www.wikidata.org/entity/Q42523)    | Atoll  | |
+[Q34763](https://www.wikidata.org/entity/Q34763)    | Peninsula  | | 
+[Q355304](https://www.wikidata.org/entity/Q355304)   | Watercourse  | |
+[Q30198](https://www.wikidata.org/entity/Q30198)    | Marsh  | [wetland=marsh](https://wiki.openstreetmap.org/wiki/Tag:wetland=marsh) |
+[Q75520](https://www.wikidata.org/entity/Q75520)    | Plateau  | |
+[Q2042028](https://www.wikidata.org/entity/Q2042028)  | Ravine  | |
+[Q631305](https://www.wikidata.org/entity/Q631305)   | Rock formation  | | 
+[Q12516](https://www.wikidata.org/entity/Q12516)    | Pyramid  | |
+[Q1076486](https://www.wikidata.org/entity/Q1076486) | Sports venue  |  |
+[Q682943](https://www.wikidata.org/entity/Q682943)   | Cricket field  | [sport=cricket](https://wiki.openstreetmap.org/wiki/Tag:sport=cricket) | 
+[Q1048525](https://www.wikidata.org/entity/Q1048525)  | Golf course  | [leisure=golf_course](https://wiki.openstreetmap.org/wiki/Tag:leisure=golf_course) |
+[Q1777138](https://www.wikidata.org/entity/Q1777138)  | Race track  | [highway=raceway](https://wiki.openstreetmap.org/wiki/Tag:highway=raceway) | 
+[Q130003](https://www.wikidata.org/entity/Q130003)   | Ski resort  | |
+[Q174782](https://www.wikidata.org/entity/Q174782)   | Town square  | [place=square](https://wiki.openstreetmap.org/wiki/Tag:place=square) |
+[Q12518](https://www.wikidata.org/entity/Q12518)    | Tower  | [building=tower](https://wiki.openstreetmap.org/wiki/Tag:building=tower), [man_made=tower](https://wiki.openstreetmap.org/wiki/Tag:man_made=tower) |
+[Q39715](https://www.wikidata.org/entity/Q39715)    | Lighthouse  | [man_made=lighthouse](https://wiki.openstreetmap.org/wiki/Tag:man_made=lighthouse) |
+[Q274153](https://www.wikidata.org/entity/Q274153)   | Water tower | [building=water_tower](https://wiki.openstreetmap.org/wiki/Tag:building=water_tower), [man_made=water_tower](https://wiki.openstreetmap.org/wiki/Tag:man_made=water_tower) |
+[Q43501](https://www.wikidata.org/entity/Q43501)    | Zoo  | [tourism=zoo](https://wiki.openstreetmap.org/wiki/Tag:tourism=zoo) | 
+[Q39614](https://www.wikidata.org/entity/Q39614)    | Cemetery  | [amenity=grave_yard](https://wiki.openstreetmap.org/wiki/Tag:amenity=grave_yard), [landuse=cemetery](https://wiki.openstreetmap.org/wiki/Tag:landuse=cemetery) |
+[Q152081](https://www.wikidata.org/entity/Q152081)   | Concentration camp  | |
+[Q1107656](https://www.wikidata.org/entity/Q1107656)  | Garden  | [leisure=garden](https://wiki.openstreetmap.org/wiki/Tag:leisure=garden) |
+[Q820477](https://www.wikidata.org/entity/Q820477)   | Mine |  | 
+[Q33837](https://www.wikidata.org/entity/Q33837) | Archipelago  | [place=archipelago](https://wiki.openstreetmap.org/wiki/Tag:place=archipelago) | 
+[Q40080](https://www.wikidata.org/entity/Q40080)    | Beach  | [natural=beach](https://wiki.openstreetmap.org/wiki/Tag:natural=beach) |
+[Q15324](https://www.wikidata.org/entity/Q15324)    | Body of water | [natural=water](https://wiki.openstreetmap.org/wiki/Tag:natural=water) | 
+[Q23397](https://www.wikidata.org/entity/Q23397)    | Lake  | [water=lake](https://wiki.openstreetmap.org/wiki/Tag:water=lake) | 
+[Q9430](https://www.wikidata.org/entity/Q9430)     | Ocean  | |
+[Q165](https://www.wikidata.org/entity/Q165)    | Sea  | |
+[Q47521](https://www.wikidata.org/entity/Q47521)    | Stream  | | 
+[Q12284](https://www.wikidata.org/entity/Q12284)    | Canal  | [waterway=canal](https://wiki.openstreetmap.org/wiki/Tag:waterway=canal) |
+[Q4022](https://www.wikidata.org/entity/Q4022)     | River  | [waterway=river](https://wiki.openstreetmap.org/wiki/Tag:waterway=river), [type=waterway](https://wiki.openstreetmap.org/wiki/Relation:waterway) |
+[Q185113](https://www.wikidata.org/entity/Q185113)   | Cape | [natural=cape](https://wiki.openstreetmap.org/wiki/Tag:natural=cape) | 
+[Q35509](https://www.wikidata.org/entity/Q35509)    | Cave  | [natural=cave_entrance](https://wiki.openstreetmap.org/wiki/Tag:natural=cave_entrance) | 
+[Q8514](https://www.wikidata.org/entity/Q8514)     | Desert  | | 
+[Q4421](https://www.wikidata.org/entity/Q4421)     | Forest  | [natural=wood](https://wiki.openstreetmap.org/wiki/Tag:natural=wood) |
+[Q35666](https://www.wikidata.org/entity/Q35666)    | Glacier  | [natural=glacier](https://wiki.openstreetmap.org/wiki/Tag:natural=glacier) |
+[Q177380](https://www.wikidata.org/entity/Q177380)   | Hot spring | | 
+[Q8502](https://www.wikidata.org/entity/Q8502)     | Mountain  | [natural=peak](https://wiki.openstreetmap.org/wiki/Tag:natural=peak) | 
+[Q133056](https://www.wikidata.org/entity/Q133056)   | Mountain pass  | | 
+[Q46831](https://www.wikidata.org/entity/Q46831)    | Mountain range  | |
+[Q39816](https://www.wikidata.org/entity/Q39816)    | Valley  | [natural=valley](https://wiki.openstreetmap.org/wiki/Tag:natural=valley) |
+[Q8072](https://www.wikidata.org/entity/Q8072)     | Volcano  | [natural=volcano](https://wiki.openstreetmap.org/wiki/Tag:natural=volcano) |
+[Q43229](https://www.wikidata.org/entity/Q43229)    | Organization  |  | 
+[Q327333](https://www.wikidata.org/entity/Q327333)   | Government agency  | [office=government](https://wiki.openstreetmap.org/wiki/Tag:office=government)|
+[Q22698](https://www.wikidata.org/entity/Q22698)    | Park | [leisure=park](https://wiki.openstreetmap.org/wiki/Tag:leisure=park) | 
+[Q159313](https://www.wikidata.org/entity/Q159313)   | Urban agglomeration | |
+[Q177634](https://www.wikidata.org/entity/Q177634)   | Community  | |
+[Q5107](https://www.wikidata.org/entity/Q5107)    | Continent | [place=continent](https://wiki.openstreetmap.org/wiki/Tag:place=continent) |
+[Q6256](https://www.wikidata.org/entity/Q6256)     | Country  | [place=country](https://wiki.openstreetmap.org/wiki/Tag:place=country) | 
+[Q75848](https://www.wikidata.org/entity/Q75848)    | Gated community | | 
+[Q3153117](https://www.wikidata.org/entity/Q3153117) | Intercommunality  | |
+[Q82794](https://www.wikidata.org/entity/Q82794)    | Region  | | 
+[Q56061](https://www.wikidata.org/entity/Q56061)    | Administrative division  | [boundary=administrative](https://wiki.openstreetmap.org/wiki/Tag:boundary=administrative)  | 
+[Q665487](https://www.wikidata.org/entity/Q665487)   | Diocese | | 
+[Q4976993](https://www.wikidata.org/entity/Q4976993)  | Parish | [boundary=civil_parish](https://wiki.openstreetmap.org/wiki/Tag:boundary=civil_parish) |
+[Q194203](https://www.wikidata.org/entity/Q194203)   | Arrondissements of France  | |
+[Q91028](https://www.wikidata.org/entity/Q91028)    | Arrondissements of Belgium  | | 
+[Q3623867](https://www.wikidata.org/entity/Q3623867)  | Arrondissements of Benin  | | 
+[Q2311958](https://www.wikidata.org/entity/Q2311958)  | Canton (country subdivision) | [political_division=canton](https://wiki.openstreetmap.org/wiki/FR:Cantons_in_France) |
+[Q643589](https://www.wikidata.org/entity/Q643589)   | Department |  | 
+[Q202216](https://www.wikidata.org/entity/Q202216)   | Overseas department and region  | |
+[Q149621](https://www.wikidata.org/entity/Q149621)   | District  | [place=district](https://wiki.openstreetmap.org/wiki/Tag:place=district) |
+[Q15243209](https://www.wikidata.org/wiki/Q15243209) | Historic district  | |
+[Q5144960](https://www.wikidata.org/entity/Q5144960)  | Microregion  | | 
+[Q15284](https://www.wikidata.org/entity/Q15284)    | Municipality  | |
+[Q515716](https://www.wikidata.org/entity/Q515716)   | Prefecture  | |
+[Q34876](https://www.wikidata.org/entity/Q34876)    | Province  | |
+[Q3191695](https://www.wikidata.org/entity/Q3191695)  | Regency (Indonesia)  | |
+[Q1970725](https://www.wikidata.org/entity/Q1970725)  | Natural region  | |
+[Q486972](https://www.wikidata.org/entity/Q486972)   | Human settlement  | | 
+[Q515](https://www.wikidata.org/entity/Q515)      | City  | [place=city](https://wiki.openstreetmap.org/wiki/Tag:place=city) |
+[Q5119](https://www.wikidata.org/entity/Q5119)     | Capital city | [capital=yes](https://wiki.openstreetmap.org/wiki/Key:capital) |
+[Q4286337](https://www.wikidata.org/entity/Q4286337)  | City district  | | 
+[Q1394476](https://www.wikidata.org/entity/Q1394476)  | Civil township  | | 
+[Q1115575](https://www.wikidata.org/entity/Q1115575)  | Civil parish  | [designation=civil_parish](https://wiki.openstreetmap.org/wiki/Tag:designation=civil_parish) |
+[Q5153984](https://www.wikidata.org/entity/Q5153984)  | Commune-level subdivisions  | |
+[Q123705](https://www.wikidata.org/entity/Q123705)   | Neighbourhood  | [place=neighbourhood](https://wiki.openstreetmap.org/wiki/Tag:place=neighbourhood) |
+[Q1500350](https://www.wikidata.org/entity/Q1500350)  | Townships of China  | |
+[Q17343829](https://www.wikidata.org/entity/Q17343829)           | Unincorporated Community  | |
+[Q3957](https://www.wikidata.org/entity/Q3957)     | Town  | [place=town](https://wiki.openstreetmap.org/wiki/Tag:place=town) | 
+[Q532](https://www.wikidata.org/entity/Q532)      | Village  | [place=village](https://wiki.openstreetmap.org/wiki/Tag:place=village) |
+[Q5084](https://www.wikidata.org/entity/Q5084)     | Hamlet   | [place=hamlet](https://wiki.openstreetmap.org/wiki/Tag:place=hamlet) | 
+[Q7275](https://www.wikidata.org/entity/Q7275)     | State  | | 
+[Q79007](https://www.wikidata.org/entity/Q79007)    | Street  | |
+[Q473972](https://www.wikidata.org/entity/Q473972)   | Protected area  | [boundary=protected_area](https://wiki.openstreetmap.org/wiki/Tag:boundary=protected_area) |
+[Q1377575](https://www.wikidata.org/entity/Q1377575)  | Wildlife refuge  | | 
+[Q1410668](https://www.wikidata.org/entity/Q1410668)  | National Wildlife Refuge  | [protection_title=National Wildlife Refuge](ownership=national), [ownership=national](https://wiki.openstreetmap.org/wiki/Tag:ownership=national)|
+[Q9259](https://www.wikidata.org/entity/Q9259)     | World Heritage Site  | |
+
+---
+
+### Future Work
+
+The Wikidata improvements to Nominatim can be further enhanced by:
+
+- continuing to add new Wikidata links to OSM objects
+- increasing the number of place types accounted for in the wikipedia_articles table
+- working to use place types in the wikipedia_article matching process

data/us_postcode_table.sql

@@ -0,0 +1,16 @@
+SET statement_timeout = 0;
+SET client_encoding = 'UTF8';
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+
+SET search_path = public, pg_catalog;
+
+SET default_tablespace = '';
+
+SET default_with_oids = false;
+
+CREATE TABLE us_postcode (
+    postcode text,
+    x double precision,
+    y double precision
+);

docs/CMakeLists.txt

@@ -18,6 +18,7 @@ ADD_CUSTOM_TARGET(doc
    COMMAND ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/data-sources/gb-postcodes/README.md ${CMAKE_CURRENT_BINARY_DIR}/data-sources/GB-Postcodes.md
    COMMAND ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/data-sources/country-grid/README.md ${CMAKE_CURRENT_BINARY_DIR}/data-sources/Country-Grid.md
    COMMAND ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/data-sources/country-grid/mexico.quad.png ${CMAKE_CURRENT_BINARY_DIR}/data-sources/mexico.quad.png
+   COMMAND ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/data-sources/wikipedia-wikidata/README.md  ${CMAKE_CURRENT_BINARY_DIR}/data-sources/Wikipedia-Wikidata.md
    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Centos-7.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Centos-7.md
    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-16.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-16.md
    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bash2md.sh ${PROJECT_SOURCE_DIR}/vagrant/Install-on-Ubuntu-18.sh ${CMAKE_CURRENT_BINARY_DIR}/appendix/Install-on-Ubuntu-18.md

docs/admin/Faq.md

@@ -24,14 +24,14 @@ If the reported rank is 26 or higher, you can also safely add `--index-noanalyse
 
 ### PHP "open_basedir restriction in effect" warnings
 
-    `PHP Warning:  file_get_contents(): open_basedir restriction in effect.`
+    PHP Warning:  file_get_contents(): open_basedir restriction in effect.
 
-You need to adjust the [open_basedir](http://www.php.net/manual/en/ini.core.php#ini.open-basedir) setting
+You need to adjust the [open_basedir](https://www.php.net/manual/en/ini.core.php#ini.open-basedir) setting
 in your PHP configuration (`php.ini file`). By default this setting may look like this:
 
     open_basedir = /srv/http/:/home/:/tmp/:/usr/share/pear/
 
-Either add reported directories to the list or disable this setting temporarily by 
+Either add reported directories to the list or disable this setting temporarily by
 dding ";" at the beginning of the line. Don't forget to enable this setting again
 once you are done with the PHP command line operations.
 
@@ -44,9 +44,9 @@ The Apache log may contain lots of PHP warnings like this:
 You should set the default time zone as instructed in the warning in
 your `php.ini` file. Find the entry about timezone and set it to
 something like this:
-  
+
     ; Defines the default timezone used by the date functions
-    ; http://php.net/date.timezone
+    ; https://php.net/date.timezone
     date.timezone = 'America/Denver'
 
 Or
@@ -66,6 +66,14 @@ server development libraries (`postgresql-server-dev-9.5` on Ubuntu)
 and recompile (`cmake .. && make`).
 
 
+## I see the error "ERROR: permission denied for language c"
+
+`nominatim.so`, written in C, is required to be installed on the database
+server. Some managed database (cloud) services like Amazon RDS do not allow
+this. There is currently no work-around other than installing a database
+on a non-managed machine.
+
+
 ### I see the error: "function transliteration(text) does not exist"
 
 Reinstall the nominatim functions with `setup.php --create--functions`
@@ -83,11 +91,11 @@ vboxfs.
 
 ### nominatim UPDATE failed: ERROR: buffer 179261 is not owned by resource owner Portal
 
-Several users [reported this](https://github.com/openstreetmap/Nominatim/issues/1168) during the initial import of the database. It's 
-something Postgresql internal Nominatim doesn't control. And Postgresql forums
+Several users [reported this](https://github.com/openstreetmap/Nominatim/issues/1168) during the initial import of the database. It's
+something PostgreSQL internal Nominatim doesn't control. And PostgreSQL forums
 suggest it's threading related but definitely some kind of crash of a process.
 Users reported either rebooting the server, different hardware or just trying
-the import again worked. 
+the import again worked.
 
 ### The website shows: "Could not get word tokens"
 
@@ -140,7 +148,7 @@ Example error message
    CONTEXT: PL/pgSQL function make_standard_name(text) line 5 at assignment]
 ```
 
-The Postgresql database, i.e. user postgres, needs to have access to that file.
+The PostgreSQL database, i.e. user `postgres`, needs to have access to that file.
 
 The permission need to be read & executable by everybody, e.g.
 
@@ -155,7 +163,7 @@ When running SELinux, make sure that the
 
 ### Setup.php fails with "DB Error: extension not found"
 
-Make sure you have the Postgres extensions hstore and postgis installed.
+Make sure you have the PostgreSQL extensions "hstore" and "postgis" installed.
 See the installation instruction for a full list of required packages.
 
 
@@ -165,7 +173,7 @@ See the installation instruction for a full list of required packages.
 
 The message is a bit misleading as PHP needs to load the file `DB.php` and
 instead re-loads Nominatim's `db.php`. To solve this make sure you
-have the [Pear module 'DB'](http://pear.php.net/package/DB/) installed.
+have the [Pear module 'DB'](https://pear.php.net/package/DB/) installed.
 
     sudo pear install DB
 
@@ -190,7 +198,7 @@ For updates you need to download the change files for each country
 once per day and apply them **separately** using
 
     ./utils/update.php --import-diff <filename> --index
-    
+
 See [this issue](https://github.com/openstreetmap/Nominatim/issues/60#issuecomment-18679446)
 for a script that runs the updates using osmosis.
 

docs/admin/Import-and-Update.md

@@ -36,7 +36,7 @@ the directory exists. There should be at least 40GB of free space.
 ### Wikipedia rankings
 
 Wikipedia can be used as an optional auxiliary data source to help indicate
-the importance of osm features. Nominatim will work without this information
+the importance of OSM features. Nominatim will work without this information
 but it will improve the quality of the results if this is installed.
 This data is available as a binary download:
 
@@ -45,18 +45,20 @@ This data is available as a binary download:
     wget https://www.nominatim.org/data/wikipedia_redirect.sql.bin
 
 Combined the 2 files are around 1.5GB and add around 30GB to the install
-size of nominatim. They also increase the install time by an hour or so.
+size of Nominatim. They also increase the install time by an hour or so.
 
 *NOTE:* you'll need to download the Wikipedia rankings before performing
 the initial import of the data if you want the rankings applied to the
 loaded data.
 
-### UK postcodes
+### Great Britain, USA postcodes
 
-Nominatim can use postcodes from an external source to improve searches that involve a UK postcode. This data can be optionally downloaded: 
+Nominatim can use postcodes from an external source to improve searches that
+involve a GB or US postcode. This data can be optionally downloaded:
 
     cd $NOMINATIM_SOURCE_DIR/data
     wget https://www.nominatim.org/data/gb_postcode_data.sql.gz
+    wget https://www.nominatim.org/data/us_postcode_data.sql.gz
 
 ## Choosing the Data to Import
 
@@ -77,7 +79,7 @@ below. There are also
 
 Please be aware that some extracts are not cut exactly along the country
 boundaries. As a result some parts of the boundary may be missing which means
-that cannot compute the areas for some administrative areas.
+that Nominatim cannot compute the areas for some administrative areas.
 
 ### Dropping Data Required for Dynamic Updates
 
@@ -99,7 +101,7 @@ database or reuse the space later.
 
 If you only want to use the Nominatim database for reverse lookups or
 if you plan to use the installation only for exports to a
-[photon](http://photon.komoot.de/) database, then you can set up a database
+[photon](https://photon.komoot.de/) database, then you can set up a database
 without search indexes. Add `--reverse-only` to your setup command above.
 
 This saves about 5% of disk space.
@@ -121,7 +123,7 @@ import styles available which only read selected data:
 
 The style can be changed with the configuration `CONST_Import_Style`.
 
-To give you an idea of the impact of using the different style, the table
+To give you an idea of the impact of using the different styles, the table
 below gives rough estimates of the final database size after import of a
 2018 planet and after using the `--drop` option. It also shows the time
 needed for the import on a machine with 32GB RAM, 4 CPUS and SSDs. Note that
@@ -136,7 +138,7 @@ address   |   59h        |  500 GB    |  260 GB
 full      |   80h        |  575 GB    |  300 GB
 
 You can also customize the styles further. For an description of the
-style format see [the developement section](../develop/Import.md).
+style format see [the development section](../develop/Import.md).
 
 ## Initial import of the data
 
@@ -157,7 +159,7 @@ about the same size as the file you are importing but never more than
 2/3 of RAM available. If your machine starts swapping reduce the size.
 
 Computing word frequency for search terms can improve the performance of
-forward geocoding in particular under high load as it helps Postgres' query
+forward geocoding in particular under high load as it helps PostgreSQL's query
 planner to make the right decisions. To recompute word counts run:
 
 ```sh
@@ -186,16 +188,16 @@ address set to complement the OSM house number data in the US. You can add
 TIGER data to your own Nominatim instance by following these steps. The
 entire US adds about 10GB to your database.
 
-  1. Get preprocessed TIGER 2018 data and unpack it into the
+  1. Get preprocessed TIGER 2019 data and unpack it into the
      data directory in your Nominatim sources:
 
         cd Nominatim/data
-        wget https://nominatim.org/data/tiger2018-nominatim-preprocessed.tar.gz
-        tar xf tiger2018-nominatim-preprocessed.tar.gz
+        wget https://nominatim.org/data/tiger2019-nominatim-preprocessed.tar.gz
+        tar xf tiger2019-nominatim-preprocessed.tar.gz
 
     `data-source/us-tiger/README.md` explains how the data got preprocessed.
 
-  2. Import the data into your Nominatim database: 
+  2. Import the data into your Nominatim database:
 
         ./utils/setup.php --import-tiger-data
 
@@ -212,20 +214,20 @@ entire US adds about 10GB to your database.
 
 ## Updates
 
-There are many different possibilities to update your Nominatim database.
+There are many different ways to update your Nominatim database.
 The following section describes how to keep it up-to-date with Pyosmium.
 For a list of other methods see the output of `./utils/update.php --help`.
 
 #### Installing the newest version of Pyosmium
 
-It is recommended to install Pyosmium via pip. Run (as the same user who
-will later run the updates):
+It is recommended to install Pyosmium via pip. Make sure to use python3.
+Run (as the same user who will later run the updates):
 
 ```sh
-pip install --user osmium
+pip3 install --user osmium
 ```
 
-Nominatim needs a tool called `pyosmium-get-updates`, which comes with
+Nominatim needs a tool called `pyosmium-get-updates` which comes with
 Pyosmium. You need to tell Nominatim where to find it. Add the
 following line to your `settings/local.php`:
 
@@ -241,7 +243,7 @@ to update using the global minutely diffs.
 
 If you want a different update source you will need to add some settings
 to `settings/local.php`. For example, to use the daily country extracts
-diffs for Ireland from geofabrik add the following:
+diffs for Ireland from Geofabrik add the following:
 
     // base URL of the replication service
     @define('CONST_Replication_Url', 'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates');
@@ -257,7 +259,7 @@ To set up the update process now run the following command:
 It outputs the date where updates will start. Recheck that this date is
 what you expect.
 
-The --init-updates command needs to be rerun whenever the replication service
+The `--init-updates` command needs to be rerun whenever the replication service
 is changed.
 
 #### Updating Nominatim

docs/admin/Installation.md

@@ -34,28 +34,28 @@ osm2pgsql README for additional dependencies required for compiling osm2pgsql.
 For running tests:
 
   * [behave](http://pythonhosted.org/behave/)
-  * [Psycopg2](http://initd.org/psycopg)
+  * [Psycopg2](https://initd.org/psycopg)
   * [nose](https://nose.readthedocs.io)
   * [phpunit](https://phpunit.de)
 
 For running Nominatim:
 
-  * [PostgreSQL](http://www.postgresql.org) (9.1 or later)
-  * [PostGIS](http://postgis.refractions.net) (2.0 or later)
-  * [PHP](http://php.net) (5.4 or later)
+  * [PostgreSQL](https://www.postgresql.org) (9.3 or later)
+  * [PostGIS](https://postgis.org) (2.2 or later)
+  * [PHP](https://php.net) (7.0 or later)
   * PHP-pgsql
   * PHP-intl (bundled with PHP)
-  * [PEAR::DB](http://pear.php.net/package/DB)
+  * [PEAR::DB](https://pear.php.net/package/DB)
   * a webserver (apache or nginx are recommended)
 
 For running continuous updates:
 
-  * [pyosmium](http://osmcode.org/pyosmium/)
+  * [pyosmium](https://osmcode.org/pyosmium/) (with Python 3)
 
 ### Hardware
 
 A minimum of 2GB of RAM is required or installation will fail. For a full
-planet import 32GB of RAM or more strongly are recommended.
+planet import 32GB of RAM or more are strongly recommended.
 
 For a full planet install you will need at least 700GB of hard disk space
 (take into account that the OSM database is growing fast). SSD disks

docs/admin/Migration.md

@@ -3,9 +3,25 @@
 This page describes database migrations necessary to update existing databases
 to newer versions of Nominatim.
 
-SQL statements should be executed from the postgres commandline. Execute
+SQL statements should be executed from the PostgreSQL commandline. Execute
 `psql nominatim` to enter command line mode.
 
+## 3.3.0 -> 3.4.0
+
+### Reorganisation of location_area_country table
+
+The table `location_area_country` has been optimized. You need to switch to the
+new format when you run updates. While updates are disabled, run the following
+SQL commands:
+
+```sql
+CREATE TABLE location_area_country_new AS
+  SELECT place_id, country_code, geometry FROM location_area_country;
+DROP TABLE location_area_country;
+ALTER TABLE location_area_country_new RENAME TO location_area_country;
+CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry);
+CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id);
+```
 
 ## 3.2.0 -> 3.3.0
 

docs/api/Details.md

@@ -24,7 +24,7 @@ but the `class` parameter is left out, then one of the places will be chosen
 at random and displayed.
 
 ```
-  https://nominatim.openstreetmap.org/details?placeid=<value>
+  https://nominatim.openstreetmap.org/details?place_id=<value>
 ```
 
 Placeids are assigned sequentially during Nominatim data import. The id for a place is different between Nominatim installation (servers) and changes when data gets reimported. Therefore it can't be used as permanent id and shouldn't be used in bug reports.
@@ -40,7 +40,7 @@ See [Place Output Formats](Output.md) for details on each format. (Default: html
 
 * `json_callback=<string>`
 
-Wrap json output in a callback function (JSONP) i.e. `<string>(<json>)`.
+Wrap JSON output in a callback function (JSONP) i.e. `<string>(<json>)`.
 Only has an effect for JSON output formats.
 
 * `pretty=[0|1]`

docs/api/Faq.md

@@ -7,11 +7,11 @@
 Nominatim computes the address from two sources in the OpenStreetMap data:
 from administrative boundaries and from place nodes. Boundaries are the more
 useful source. They precisely describe an area. So it is very clear for
-Nominatim if a point belongs to an area of not. Place nodes are more complicated.
-These are only points without any precise extend. So Nominatim has to take a
-guess and assume that an address belongs to the closest place nose it can find.
+Nominatim if a point belongs to an area or not. Place nodes are more complicated.
+These are only points without any precise extent. So Nominatim has to take a
+guess and assume that an address belongs to the closest place node it can find.
 In an ideal world, Nominatim would not need the place nodes but there are
-many places on earth where there are not precise boundaries available for
+many places on earth where there are no precise boundaries available for
 all parts that make up an address. This is in particular true for the more
 local address parts, like villages and suburbs. Therefore it is not possible
 to completely dismiss place nodes. And sometimes they sneak in where they
@@ -21,7 +21,7 @@ As a OpenStreetMap mapper, you can improve the situation in two ways: if you
 see a place node for which already an administrative area exists, then you
 should _link_ the two by adding the node with a 'label' role to the boundary
 relation. If there is no administrative area, you can add the approximate
-extend of the place and tag it place=<something> as well.
+extent of the place and tag it place=<something> as well.
 
 #### 2. When doing reverse search, the address details have parts that don't contain the point I was looking up.
 
@@ -30,7 +30,7 @@ Reverse does not give you the address of the point you asked for. Reverse
 returns the closest object to the point you asked for and then returns the
 address of that object. Now, if you are close to a border, then the closest
 object may be across that border. When Nominatim then returns the address,
-contains the county/state/country across the border.
+it contains the county/state/country across the border.
 
 #### 3. I get different counties/states/countries when I change the zoom parameter in the reverse query. How is that possible?
 
@@ -45,12 +45,12 @@ sometimes the other for the closest point.
 
 Nominatim assigns each map feature one country. Those outside any administrative
 boundaries are assigned a special no-country. Continents or other super-national
-administrations (e.g. European Union, NATO, Custom unions) are not supported, 
+administrations (e.g. European Union, NATO, Custom unions) are not supported,
 see also [Administrative Boundary](https://wiki.openstreetmap.org/wiki/Tag:boundary%3Dadministrative#Super-national_administrations).
 
 #### 5. Can you return the timezone?
 
-See this separate OpenStreetMap-based project [Timezone Boundary Builder](https://github.com/evansiroky/timezone-boundary-builder)
+See this separate OpenStreetMap-based project [Timezone Boundary Builder](https://github.com/evansiroky/timezone-boundary-builder).
 
 #### 6. I want to download a list of streets/restaurants of a city/region
 

docs/api/Lookup.md

@@ -19,13 +19,13 @@ Additional optional parameters are explained below.
 
 ### Output format
 
-* `format=[html|xml|json|jsonv2|geojson|geocodejson]`
+* `format=[xml|json|jsonv2|geojson|geocodejson]`
 
 See [Place Output Formats](Output.md) for details on each format. (Default: xml)
 
 * `json_callback=<string>`
 
-Wrap json output in a callback function (JSONP) i.e. `<string>(<json>)`.
+Wrap JSON output in a callback function (JSONP) i.e. `<string>(<json>)`.
 Only has an effect for JSON output formats.
 
 ### Output details

docs/api/Output.md

@@ -1,6 +1,6 @@
 # Place Output
 
-The [\reverse](Reverse.md), [\search](Search.md) and [\lookup](Lookup.md)
+The [/reverse](Reverse.md), [/search](Search.md) and [/lookup](Lookup.md)
 API calls produce very similar output which is explained in this section.
 There is one section for each format which is selectable via the `format`
 parameter.
@@ -70,7 +70,7 @@ This is the same as the JSON format with two changes:
 
 ### GeoJSON
 
-This format follows the [RFC7946](http://geojson.org). Every feature includes
+This format follows the [RFC7946](https://geojson.org). Every feature includes
 a bounding box (`bbox`).
 
 The feature list has the following fields:
@@ -83,7 +83,7 @@ The feature list has the following fields:
  * `importance` - computed importance rank
  * `icon` - link to class icon (if available)
  * `address` - dictionary of address details (only with `addressdetails=1`)
- * `extratags` - dictionary with additional useful tags like website or maxspeed
+ * `extratags` - dictionary with additional useful tags like `website` or `maxspeed`
    (only with `extratags=1`)
  * `namedetails` - dictionary with full list of available names including ref etc.
 
@@ -120,7 +120,7 @@ formats depending on the API call.
 
 ```
 <reversegeocode timestamp="Sat, 11 Aug 18 11:53:21 +0000"
-                attribution="Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright"
+                attribution="Data © OpenStreetMap contributors, ODbL 1.0. https://www.openstreetmap.org/copyright"
                 querystring="lat=48.400381&lon=11.745876&zoom=5&format=xml">
   <result place_id="179509537" osm_type="relation" osm_id="2145268" ref="BY"
           lat="48.9467562" lon="11.4038717"
@@ -154,7 +154,7 @@ The place information can be found in the `result` element. The attributes of th
  * `lat`, `lon` - latitude and longitude of the centroid of the object
  * `boundingbox` - comma-separated list of corner coordinates
 
-The full address address of the result can be found in the content of the
+The full address of the result can be found in the content of the
 `result` element as a comma-separated list.
 
 Additional information requested with `addressdetails=1`, `extratags=1` and
@@ -164,12 +164,12 @@ Additional information requested with `addressdetails=1`, `extratags=1` and
 
 ```
 <searchresults timestamp="Sat, 11 Aug 18 11:55:35 +0000"
-               attribution="Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright"
+               attribution="Data © OpenStreetMap contributors, ODbL 1.0. https://www.openstreetmap.org/copyright"
                querystring="london" polygon="false" exclude_place_ids="100149"
                more_url="https://nominatim.openstreetmap.org/search.php?q=london&addressdetails=1&extratags=1&exclude_place_ids=100149&format=xml&accept-language=en-US%2Cen%3Bq%3D0.7%2Cde%3Bq%3D0.3">
   <place place_id="100149" osm_type="node" osm_id="107775" place_rank="15"
          boundingbox="51.3473219,51.6673219,-0.2876474,0.0323526" lat="51.5073219" lon="-0.1276474"
-         display_name="London, Greater London, England, SW1A 2DU, United Kingdom" 
+         display_name="London, Greater London, England, SW1A 2DU, United Kingdom"
          class="place" type="city" importance="0.9654895765402"
          icon="https://nominatim.openstreetmap.org/images/mapicons/poi_place_city.p.20.png">
     <extratags>
@@ -232,7 +232,7 @@ permanent for later use.
 
 The combination `osm_type`+`osm_id` is slighly better but remember in
 OpenStreetMap mappers can delete, split, recreate places (and those
-get a new `osm_id`), there is no link between those old and new id.
+get a new `osm_id`), there is no link between those old and new ids.
 Places can also change their meaning without changing their `osm_id`,
 e.g. when a restaurant is retagged as supermarket. For a more in-depth
 discussion see [Permanent ID](https://wiki.openstreetmap.org/wiki/Permanent_ID).

docs/api/Reverse.md

@@ -22,7 +22,7 @@ There are two ways how the requested location can be specified:
 
     A specific OSM node(N), way(W) or relation(R) to return an address for.
 
-In both cases exactly one object is returned. The two input paramters cannot
+In both cases exactly one object is returned. The two input parameters cannot
 be used at the same time. Both accept the additional optional parameters listed
 below.
 
@@ -34,7 +34,7 @@ See [Place Output Formats](Output.md) for details on each format. (Default: html
 
 * `json_callback=<string>`
 
-Wrap json output in a callback function ([JSONP](https://en.wikipedia.org/wiki/JSONP)) i.e. `<string>(<json>)`.
+Wrap JSON output in a callback function ([JSONP](https://en.wikipedia.org/wiki/JSONP)) i.e. `<string>(<json>)`.
 Only has an effect for JSON output formats.
 
 ### Output details
@@ -135,7 +135,7 @@ This overrides the specified machine readable format. (Default: 0)
       <postcode>B72</postcode>
       <country>United Kingdom</country>
       <country_code>gb</country_code>
-    </addressparts>   
+    </addressparts>
   </reversegeocode>
 ```
 
@@ -146,7 +146,7 @@ This overrides the specified machine readable format. (Default: 0)
 ```json
 {
   "place_id":"134140761",
-  "licence":"Data © OpenStreetMap contributors, ODbL 1.0. http:\/\/www.openstreetmap.org\/copyright",
+  "licence":"Data © OpenStreetMap contributors, ODbL 1.0. https:\/\/www.openstreetmap.org\/copyright",
   "osm_type":"way",
   "osm_id":"280940520",
 "lat":"-34.4391708",

docs/api/Search.md

@@ -1,6 +1,6 @@
 # Search queries
 
-The search API allows to look up a location from a textual description.
+The search API allows you to look up a location from a textual description.
 Nominatim supports structured as well as free-form search queries.
 
 The search query may also contain
@@ -46,7 +46,7 @@ In this form, the query may be given through two different sets of parameters:
     Structured requests are faster but are less robust against alternative
     OSM tagging schemas. **Do not combine with** `q=<query>` **parameter**.
 
-All three query forms accept the additional paramters listed below.
+All three query forms accept the additional parameters listed below.
 
 ### Output format
 
@@ -56,7 +56,7 @@ See [Place Output Formats](Output.md) for details on each format. (Default: html
 
 * `json_callback=<string>`
 
-Wrap json output in a callback function ([JSONP](https://en.wikipedia.org/wiki/JSONP)) i.e. `<string>(<json>)`.
+Wrap JSON output in a callback function ([JSONP](https://en.wikipedia.org/wiki/JSONP)) i.e. `<string>(<json>)`.
 Only has an effect for JSON output formats.
 
 ### Output details
@@ -112,7 +112,8 @@ Limit the number of returned results. (Default: 10, Maximum: 50)
 * `viewbox=<x1>,<y1>,<x2>,<y2>`
 
 The preferred area to find search results. Any two corner points of the box
-are accepted in any order as long as they span a real box.
+are accepted in any order as long as they span a real box. `x` is longitude,
+`y` is latitude.
 
 
 * `bounded=[0|1]`
@@ -175,12 +176,12 @@ This overrides the specified machine readable format. (Default: 0)
 
 ```xml
   <searchresults timestamp="Sat, 07 Nov 09 14:42:10 +0000" querystring="135 pilkington, avenue birmingham" polygon="true">
-    <place 
-      place_id="1620612" osm_type="node" osm_id="452010817" 
-      boundingbox="52.548641204834,52.5488433837891,-1.81612110137939,-1.81592094898224" 
-      polygonpoints="[['-1.81592098644987','52.5487429714954'],['-1.81592290792183','52.5487234624632'],...]" 
-      lat="52.5487429714954" lon="-1.81602098644987" 
-      display_name="135, Pilkington Avenue, Wylde Green, City of Birmingham, West Midlands (county), B72, United Kingdom" 
+    <place
+      place_id="1620612" osm_type="node" osm_id="452010817"
+      boundingbox="52.548641204834,52.5488433837891,-1.81612110137939,-1.81592094898224"
+      polygonpoints="[['-1.81592098644987','52.5487429714954'],['-1.81592290792183','52.5487234624632'],...]"
+      lat="52.5487429714954" lon="-1.81602098644987"
+      display_name="135, Pilkington Avenue, Wylde Green, City of Birmingham, West Midlands (county), B72, United Kingdom"
       class="place" type="house">
       <house_number>135</house_number>
       <road>Pilkington Avenue</road>
@@ -237,7 +238,7 @@ This overrides the specified machine readable format. (Default: 0)
 
 ##### JSON with address details
 
-[https://nominatim.openstreetmap.org/?format=json&addressdetails=1&q=bakery+in+berlin+wedding&format=json&limit=1](https://nominatim.openstreetmap.org/?format=json&addressdetails=1&q=bakery+in+berlin+wedding&format=json&limit=1)
+[https://nominatim.openstreetmap.org/?addressdetails=1&q=bakery+in+berlin+wedding&format=json&limit=1](https://nominatim.openstreetmap.org/?addressdetails=1&q=bakery+in+berlin+wedding&format=json&limit=1)
 
 ```json
     {

docs/develop/Documentation.md

@@ -0,0 +1,36 @@
+# Documentation Pages
+
+The [Nominatim documentation](https://nominatim.org/release-docs/develop/) is built using the [MkDocs](https://www.mkdocs.org/) static site generation framework. The master branch is automatically deployed every night on under [https://nominatim.org/release-docs/develop/]()
+
+To preview local changes:
+
+1. Install MkDocs
+
+   ```
+   pip3 install --user mkdocs 
+   ```
+
+
+2. In build directory run
+
+   ```
+   make doc
+   INFO - Cleaning site directory
+   INFO - Building documentation to directory: /home/vagrant/build/site-html 
+   ```
+
+   This runs `mkdocs build` plus extra transformion of some files and adds symlinks (see `CMakeLists.txt` for the exact steps).
+
+
+3. Start webserver for local testing
+
+   ```
+   mkdocs serve
+   [server:296] Serving on http://127.0.0.1:8000
+   [handlers:62] Start watching changes
+   ```
+
+   If you develop inside a Vagrant virtual machine:
+   * add port forwarding to your Vagrantfile, e.g. `config.vm.network "forwarded_port", guest: 8000, host: 8000`
+   * use `mkdocs serve --dev-addr 0.0.0.0:8000` because the default localhost
+      IP does not get forwarded.

docs/develop/Import.md

@@ -24,7 +24,7 @@ with multiple tags that may constitute a principal tag. Take for example a
 motorway bridge. In OSM, this would be a way which is tagged with
 `highway=motorway` and `bridge=yes`. This way would appear in the `place` table
 once with `class` of `highway` and once with a `class` of `bridge`. Thus the
-*uique key* for `place` is (`osm_type`, `osm_id`, `class`).
+*unique key* for `place` is (`osm_type`, `osm_id`, `class`).
 
 ## Configuring the Import
 
@@ -55,8 +55,8 @@ suffix match can be defined similarly with a string that starts with a `*`. Any
 other string constitutes an exact match.
 
 The second part of the rules defines a list of values and the properties that
-apply to a successful match. Value strings may be either empty, which again
-means that thy match against any value, or describe an exact match. Prefix
+apply to a successful match. Value strings may be either empty, which
+means that they match any value, or describe an exact match. Prefix
 or suffix matching of values is not possible.
 
 For a rule to match, it has to find a valid combination of keys and values. The
@@ -66,7 +66,7 @@ The rules in a configuration file are processed sequentially and the first
 match for each tag wins.
 
 A rule where key and value are the empty string is special. This defines the
-fallback when none of the rules matches. The fallback is always used as a last
+fallback when none of the rules match. The fallback is always used as a last
 resort when nothing else matches, no matter where the rule appears in the file.
 Defining multiple fallback rules is not allowed. What happens in this case,
 is undefined.
@@ -121,17 +121,17 @@ One or more of the following properties may be given for each tag:
 
 * `address`
 
-    At tag to the list of address tags. If the tag starts with `addr:` or
+    Add tag to the list of address tags. If the tag starts with `addr:` or
     `is_in:`, then this prefix is cut off before adding it to the list.
 
 * `postcode`
 
-    At the value as a postcode to the address tags. If multiple tags are
+    Add the value as a postcode to the address tags. If multiple tags are
     candidate for postcodes, one wins out and the others are dropped.
 
 * `country`
 
-    At the value as a country code to the address tags. The value must be a
+    Add the value as a country code to the address tags. The value must be a
     two letter country code, otherwise it is ignored. If there are multiple
     tags that match, then one wins out and the others are dropped.
 

docs/develop/Ranking.md

@@ -21,7 +21,7 @@ Usually only administrative boundaries and place nodes and areas are
 eligible to be part of an address. All other objects have an address rank
 of 0.
 
-Note that the search rank of a place place a role in the address computation
+Note that the search rank of a place plays a role in the address computation
 as well. When collecting the places that should make up the address parts
 then only places are taken into account that have a lower address rank than
 the search rank of the base object.
@@ -37,7 +37,7 @@ into the database. There are a few hard-coded rules for the assignment:
     * highway nodes
     * landuse that is not an area
 
-Other than that, the ranks can be freely assigned via the json file
+Other than that, the ranks can be freely assigned via the JSON file
 defined with `CONST_Address_Level_Config` according to their type and
 the country they are in.
 
@@ -78,12 +78,13 @@ definition is used as a fallback, when nothing more specific for a given
 country exists.
 
 `tags` contains the ranks for key/value pairs. The ranks can be either a
-single number, in which case they are to search and address rank, or a tuple
+single number, in which case they are the search and address rank, or an array
 of search and address rank (in that order). The value may be left empty.
 Then the rank is used when no more specific value is found for the given
 key.
 
-Countries and key/value combination may appear in multiple defintions. Just
+Countries and key/value combination may appear in multiple definitions. Just
 make sure that each combination of counrty/key/value appears only once per
 file. Otherwise the import will fail with a UNIQUE INDEX constraint violation
 on import.
+

docs/develop/overview.md

@@ -1,6 +1,6 @@
 # Basic Architecture
 
-Nominatim provides geocoding based on OpenStreetMap data. It uses a Postgresql
+Nominatim provides geocoding based on OpenStreetMap data. It uses a PostgreSQL
 database as a backend for storing the data.
 
 There are three basic parts to Nominatim's architecture: the data import,
@@ -15,10 +15,10 @@ the import can be found in the database table `place`.
 The __address computation__ or __indexing__ stage takes the data from `place`
 and adds additional information needed for geocoding. It ranks the places by
 importance, links objects that belong together and computes addresses and
-the search index. Most of this work is done in Pl/pqSQL via database triggers
+the search index. Most of this work is done in PL/pgSQL via database triggers
 and can be found in the file `sql/functions.sql`.
 
-The __search frontend__ implements the actual API. It takes queries for
-search and reverse geocoding queries from the user, looks up the data and
+The __search frontend__ implements the actual API. It takes search
+and reverse geocoding queries from the user, looks up the data and
 returns the results in the requested format. This part is written in PHP
 and can be found in the `lib/` and `website/` directories.

docs/mkdocs.yml

@@ -22,11 +22,13 @@ pages:
         - 'Overview' : 'develop/overview.md'
         - 'OSM Data Import' : 'develop/Import.md'
         - 'Place Ranking' : 'develop/Ranking.md'
+        - 'Documentation' : 'develop/Documentation.md'
     - 'External Data Sources':
         - 'Overview' : 'data-sources/overview.md'
         - 'US Census (Tiger)': 'data-sources/US-Tiger.md'
         - 'GB Postcodes': 'data-sources/GB-Postcodes.md'
         - 'Country Grid': 'data-sources/Country-Grid.md'
+        - 'Wikipedia & Wikidata': 'data-sources/Wikipedia-Wikidata.md'
     - 'Appendix':
         - 'Installation on CentOS 7' : 'appendix/Install-on-Centos-7.md'
         - 'Installation on Ubuntu 16' : 'appendix/Install-on-Ubuntu-16.md'

lib/AddressDetails.php

@@ -14,7 +14,7 @@ class AddressDetails
     public function __construct(&$oDB, $iPlaceID, $sHousenumber, $mLangPref)
     {
         if (is_array($mLangPref)) {
-            $mLangPref = 'ARRAY['.join(',', array_map('getDBQuoted', $mLangPref)).']';
+            $mLangPref = $oDB->getArraySQL($oDB->getDBQuotedList($mLangPref));
         }
 
         if (!isset($sHousenumber)) {

lib/Geocode.php

@@ -348,10 +348,7 @@ class Geocode
             $aNewPhraseSearches = array();
             $sPhraseType = $bIsStructured ? $oPhrase->getPhraseType() : '';
 
-            foreach ($oPhrase->getWordSets() as $iWordSet => $aWordset) {
-                // Too many permutations - too expensive
-                if ($iWordSet > 120) break;
-
+            foreach ($oPhrase->getWordSets() as $aWordset) {
                 $aWordsetSearches = $aSearches;
 
                 // Add all words from this wordset
@@ -641,7 +638,6 @@ class Geocode
                 }
             }
 
-            Debug::printDebugTable('Phrases', $aPhrases);
             Debug::printVar('Tokens', $aTokens);
 
             $oValidTokens = new TokenList();
@@ -686,6 +682,11 @@ class Geocode
 
                 Debug::printGroupTable('Valid Tokens', $oValidTokens->debugInfo());
 
+                foreach ($aPhrases as $oPhrase) {
+                    $oPhrase->computeWordSets($oValidTokens);
+                }
+                Debug::printDebugTable('Phrases', $aPhrases);
+
                 Debug::newSection('Search candidates');
 
                 $aGroupedSearches = $this->getGroupedSearches($aSearches, $aPhrases, $oValidTokens, $bStructuredPhrases);

lib/Phrase.php

@@ -9,7 +9,8 @@ namespace Nominatim;
  */
 class Phrase
 {
-    const MAX_DEPTH = 7;
+    const MAX_WORDSET_LEN = 20;
+    const MAX_WORDSETS = 100;
 
     // Complete phrase as a string.
     private $sPhrase;
@@ -20,13 +21,24 @@ class Phrase
     // Possible segmentations of the phrase.
     private $aWordSets;
 
+    public static function cmpByArraylen($aA, $aB)
+    {
+        $iALen = count($aA);
+        $iBLen = count($aB);
+
+        if ($iALen == $iBLen) {
+            return 0;
+        }
+
+        return ($iALen < $iBLen) ? -1 : 1;
+    }
+
 
     public function __construct($sPhrase, $sPhraseType)
     {
         $this->sPhrase = trim($sPhrase);
         $this->sPhraseType = $sPhraseType;
         $this->aWords = explode(' ', $this->sPhrase);
-        $this->aWordSets = $this->createWordSets($this->aWords, 0);
     }
 
     /**
@@ -60,10 +72,17 @@ class Phrase
      */
     public function addTokens(&$aTokens)
     {
-        foreach ($this->aWordSets as $aSet) {
-            foreach ($aSet as $sWord) {
-                $aTokens[' '.$sWord] = ' '.$sWord;
-                $aTokens[$sWord] = $sWord;
+        $iNumWords = count($this->aWords);
+
+        for ($i = 0; $i < $iNumWords; $i++) {
+            $sPhrase = $this->aWords[$i];
+            $aTokens[' '.$sPhrase] = ' '.$sPhrase;
+            $aTokens[$sPhrase] = $sPhrase;
+
+            for ($j = $i + 1; $j < $iNumWords; $j++) {
+                $sPhrase .= ' '.$this->aWords[$j];
+                $aTokens[' '.$sPhrase] = ' '.$sPhrase;
+                $aTokens[$sPhrase] = $sPhrase;
             }
         }
     }
@@ -75,44 +94,59 @@ class Phrase
      */
     public function invertWordSets()
     {
-        $this->aWordSets = $this->createInverseWordSets($this->aWords, 0);
+        foreach ($this->aWordSets as $i => $aSet) {
+            $this->aWordSets[$i] = array_reverse($aSet);
+        }
     }
 
-    private function createWordSets($aWords, $iDepth)
+    public function computeWordSets($oTokens)
     {
-        $aResult = array(array(join(' ', $aWords)));
-        $sFirstToken = '';
-        if ($iDepth < Phrase::MAX_DEPTH) {
-            while (count($aWords) > 1) {
-                $sWord = array_shift($aWords);
-                $sFirstToken .= ($sFirstToken?' ':'').$sWord;
-                $aRest = $this->createWordSets($aWords, $iDepth + 1);
-                foreach ($aRest as $aSet) {
-                    $aResult[] = array_merge(array($sFirstToken), $aSet);
+        $iNumWords = count($this->aWords);
+        // Caches the word set for the partial phrase up to word i.
+        $aSetCache = array_fill(0, $iNumWords, array());
+
+        // Initialise first element of cache. There can only be the word.
+        if ($oTokens->containsAny($this->aWords[0])) {
+            $aSetCache[0][] = array($this->aWords[0]);
+        }
+
+        // Now do the next elements using what we already have.
+        for ($i = 1; $i < $iNumWords; $i++) {
+            for ($j = $i; $j > 0; $j--) {
+                $sPartial = $j == $i ? $this->aWords[$j] : $this->aWords[$j].' '.$sPartial;
+                if (!empty($aSetCache[$j - 1]) && $oTokens->containsAny($sPartial)) {
+                    $aPartial = array($sPartial);
+                    foreach ($aSetCache[$j - 1] as $aSet) {
+                        if (count($aSet) < Phrase::MAX_WORDSET_LEN) {
+                            $aSetCache[$i][] = array_merge($aSet, $aPartial);
+                        }
+                    }
+                    if (count($aSetCache[$i]) > 2 * Phrase::MAX_WORDSETS) {
+                        usort(
+                            $aSetCache[$i],
+                            array('\Nominatim\Phrase', 'cmpByArraylen')
+                        );
+                        $aSetCache[$i] = array_slice(
+                            $aSetCache[$i],
+                            0,
+                            Phrase::MAX_WORDSETS
+                        );
+                    }
                 }
             }
+
+            // finally the current full phrase
+            $sPartial = $this->aWords[0].' '.$sPartial;
+            if ($oTokens->containsAny($sPartial)) {
+                $aSetCache[$i][] = array($sPartial);
+            }
         }
 
-        return $aResult;
+        $this->aWordSets = $aSetCache[$iNumWords - 1];
+        usort($this->aWordSets, array('\Nominatim\Phrase', 'cmpByArraylen'));
+        $this->aWordSets = array_slice($this->aWordSets, 0, Phrase::MAX_WORDSETS);
     }
 
-    private function createInverseWordSets($aWords, $iDepth)
-    {
-        $aResult = array(array(join(' ', $aWords)));
-        $sFirstToken = '';
-        if ($iDepth < Phrase::MAX_DEPTH) {
-            while (count($aWords) > 1) {
-                $sWord = array_pop($aWords);
-                $sFirstToken = $sWord.($sFirstToken?' ':'').$sFirstToken;
-                $aRest = $this->createInverseWordSets($aWords, $iDepth + 1);
-                foreach ($aRest as $aSet) {
-                    $aResult[] = array_merge(array($sFirstToken), $aSet);
-                }
-            }
-        }
-
-        return $aResult;
-    }
 
     public function debugInfo()
     {

lib/TokenList.php

@@ -55,6 +55,18 @@ class TokenList
         return isset($this->aTokens[$sWord]);
     }
 
+    /**
+     * Check if there are partial or full tokens for the given word.
+     *
+     * @param string $sWord Token word to look for.
+     *
+     * @return bool True if there is one or more token for the token word.
+     */
+    public function containsAny($sWord)
+    {
+        return isset($this->aTokens[$sWord]) || isset($this->aTokens[' '.$sWord]);
+    }
+
     /**
      * Get the list of tokens for the given token word.
      *

lib/init-website.php

@@ -21,6 +21,7 @@ function exception_handler_html($exception)
     http_response_code($exception->getCode());
     header('Content-type: text/html; charset=UTF-8');
     include(CONST_BasePath.'/lib/template/error-html.php');
+    exit();
 }
 
 function exception_handler_json($exception)
@@ -28,6 +29,7 @@ function exception_handler_json($exception)
     http_response_code($exception->getCode());
     header('Content-type: application/json; charset=utf-8');
     include(CONST_BasePath.'/lib/template/error-json.php');
+    exit();
 }
 
 function exception_handler_xml($exception)
@@ -36,17 +38,51 @@ function exception_handler_xml($exception)
     header('Content-type: text/xml; charset=utf-8');
     echo '<?xml version="1.0" encoding="UTF-8" ?>'."\n";
     include(CONST_BasePath.'/lib/template/error-xml.php');
+    exit();
+}
+
+function shutdown_exception_handler_html()
+{
+    $error = error_get_last();
+    if ($error !== null && $error['type'] === E_ERROR) {
+        exception_handler_html(new Exception($error['message'], 500));
+    }
+}
+
+function shutdown_exception_handler_xml()
+{
+    $error = error_get_last();
+    if ($error !== null && $error['type'] === E_ERROR) {
+        exception_handler_xml(new Exception($error['message'], 500));
+    }
+}
+
+function shutdown_exception_handler_json()
+{
+    $error = error_get_last();
+    if ($error !== null && $error['type'] === E_ERROR) {
+        exception_handler_json(new Exception($error['message'], 500));
+    }
 }
 
 
-function set_exception_handler_by_format($sFormat = 'html')
+function set_exception_handler_by_format($sFormat = null)
 {
-    if ($sFormat == 'html') {
+    // Multiple calls to register_shutdown_function will cause multiple callbacks
+    // to be executed, we only want the last executed. Thus we don't want to register
+    // one by default without an explicit $sFormat set.
+
+    if (!isset($sFormat)) {
         set_exception_handler('exception_handler_html');
+    } elseif ($sFormat == 'html') {
+        set_exception_handler('exception_handler_html');
+        register_shutdown_function('shutdown_exception_handler_html');
     } elseif ($sFormat == 'xml') {
         set_exception_handler('exception_handler_xml');
+        register_shutdown_function('shutdown_exception_handler_xml');
     } else {
         set_exception_handler('exception_handler_json');
+        register_shutdown_function('shutdown_exception_handler_json');
     }
 }
 // set a default

lib/log.php

@@ -36,7 +36,7 @@ function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array())
             $sUserAgent = $_SERVER['HTTP_USER_AGENT'];
         else $sUserAgent = '';
         $sSQL = 'insert into new_query_log (type,starttime,query,ipaddress,useragent,language,format,searchterm)';
-        $sSQL .= ' values ('.
+        $sSQL .= ' values (';
         $sSQL .= join(',', $oDB->getDBQuotedList(array(
             $sType,
             $hLog[0],
@@ -48,7 +48,7 @@ function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array())
             $hLog[3]
         )));
         $sSQL .= ')';
-        $oDB->query($sSQL);
+        $oDB->exec($sSQL);
     }
 
     return $hLog;
@@ -67,7 +67,7 @@ function logEnd(&$oDB, $hLog, $iNumResults)
         $sSQL .= ' where starttime = '.$oDB->getDBQuoted($hLog[0]);
         $sSQL .= ' and ipaddress = '.$oDB->getDBQuoted($hLog[1]);
         $sSQL .= ' and query = '.$oDB->getDBQuoted($hLog[2]);
-        $oDB->query($sSQL);
+        $oDB->exec($sSQL);
     }
 
     if (CONST_Log_File) {

lib/setup/SetupClass.php

@@ -106,34 +106,19 @@ class SetupFunctions
         $fPostgresVersion = $this->oDB->getPostgresVersion();
         echo 'Postgres version found: '.$fPostgresVersion."\n";
 
-        if ($fPostgresVersion < 9.01) {
-            fail('Minimum supported version of Postgresql is 9.1.');
+        if ($fPostgresVersion < 9.03) {
+            fail('Minimum supported version of Postgresql is 9.3.');
         }
 
         $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
         $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
 
-        // For extratags and namedetails the hstore_to_json converter is
-        // needed which is only available from Postgresql 9.3+. For older
-        // versions add a dummy function that returns nothing.
-        $iNumFunc = $this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'");
-
-        if ($iNumFunc == 0) {
-            $this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable");
-            warn('Postgresql is too old. extratags and namedetails API not available.');
-        }
-
-
         $fPostgisVersion = $this->oDB->getPostgisVersion();
         echo 'Postgis version found: '.$fPostgisVersion."\n";
 
-        if ($fPostgisVersion < 2.1) {
-            // Functions were renamed in 2.1 and throw an annoying deprecation warning
-            $this->pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
-            $this->pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint');
-        }
         if ($fPostgisVersion < 2.2) {
-            $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
+            echo "Minimum required Postgis version 2.2\n";
+            exit(1);
         }
 
         $i = $this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'");
@@ -154,16 +139,20 @@ class SetupFunctions
         $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
         $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz');
         $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
+        $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode_table.sql');
 
         $sPostcodeFilename = CONST_BasePath.'/data/gb_postcode_data.sql.gz';
         if (file_exists($sPostcodeFilename)) {
             $this->pgsqlRunScriptFile($sPostcodeFilename);
         } else {
-            warn('optional external UK postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
+            warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
         }
 
-        if (CONST_Use_Extra_US_Postcodes) {
-            $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
+        $sPostcodeFilename = CONST_BasePath.'/data/us_postcode_data.sql.gz';
+        if (file_exists($sPostcodeFilename)) {
+            $this->pgsqlRunScriptFile($sPostcodeFilename);
+        } else {
+            warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
         }
 
         if ($this->bNoPartitions) {
@@ -573,17 +562,15 @@ class SetupFunctions
         $sSQL .= ' GROUP BY country_code, pc';
         $this->pgExec($sSQL);
 
-        if (CONST_Use_Extra_US_Postcodes) {
-            // only add postcodes that are not yet available in OSM
-            $sSQL  = 'INSERT INTO location_postcode';
-            $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
-            $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
-            $sSQL .= '       ST_SetSRID(ST_Point(x,y),4326)';
-            $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
-            $sSQL .= '        (SELECT postcode FROM location_postcode';
-            $sSQL .= "          WHERE country_code = 'us')";
-            $this->pgExec($sSQL);
-        }
+        // only add postcodes that are not yet available in OSM
+        $sSQL  = 'INSERT INTO location_postcode';
+        $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
+        $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
+        $sSQL .= '       ST_SetSRID(ST_Point(x,y),4326)';
+        $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
+        $sSQL .= '        (SELECT postcode FROM location_postcode';
+        $sSQL .= "          WHERE country_code = 'us')";
+        $this->pgExec($sSQL);
 
         // add missing postcodes for GB (if available)
         $sSQL  = 'INSERT INTO location_postcode';
@@ -738,9 +725,7 @@ class SetupFunctions
         }
         foreach ($aDropTables as $sDrop) {
             if ($this->bVerbose) echo "Dropping table $sDrop\n";
-            $this->oDB->exec("DROP TABLE $sDrop CASCADE");
-            // ignore warnings/errors as they might be caused by a table having
-            // been deleted already by CASCADE
+            $this->oDB->exec("DROP TABLE IF EXISTS $sDrop CASCADE");
         }
 
         if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {

settings/address-levels.json

@@ -3,8 +3,8 @@
       "place" : {
           "sea" : [2, 0],
           "continent" : [2, 0],
-          "country" : [4, 4],
-          "state" : [8, 8],
+          "country" : [4, 0],
+          "state" : [8, 0],
           "region" : [18, 0],
           "county" : 12,
           "city" : 16,
@@ -25,6 +25,8 @@
           "islet" : [20, 0],
           "mountain_pass" : [20, 0],
           "neighbourhood" : 22,
+          "quarter" : 22,
+          "city_block" : 22,
           "houses" : [28, 0]
       },
       "boundary" : {
@@ -78,6 +80,26 @@
       },
       "mountain_pass" : {
           "" : [20, 0]
+      },
+      "historic" : {
+          "neighbourhood" : [30, 0]
+      }
+  }
+},
+{ "countries" : [ "de" ],
+  "tags" : {
+      "place" : {
+          "county" : [12, 0]
+      },
+      "boundary" : {
+          "administrative5" : [10, 0]
+      }
+  }
+},
+{ "countries" : [ "be" ],
+  "tags" : {
+      "boundary" : {
+          "administrative7" : [14, 0]
       }
   }
 }

settings/defaults.php

@@ -23,8 +23,6 @@ if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true);
 // term. Spaces are kept but collapsed to one standard space.
 @define('CONST_Term_Normalization_Rules', ":: NFD (); [[:Nonspacing Mark:] [:Cf:]] >;  :: lower (); [[:Punctuation:][:Space:]]+ > ' '; :: NFC ();");
 
-// Set to false to avoid importing extra postcodes for the US.
-@define('CONST_Use_Extra_US_Postcodes', true);
 /* Set to true after importing Tiger house number data for the US.
    Note: The tables must already exist or queries will throw errors.
          After changing this setting run ./utils/setup --create-functions

settings/import-address.style

@@ -73,7 +73,7 @@
     }
 },
 {
-    "keys" : ["country_code", "ISO3166-1", "is_in:country_code", "is_in_country",
+    "keys" : ["country_code", "ISO3166-1", "is_in:country_code", "is_in:country",
               "addr:country", "addr:country", "addr:country_code"],
     "values" : {
         "" : "country"

settings/import-admin.style

@@ -42,7 +42,7 @@
     }
 },
 {
-    "keys" : ["country_code", "ISO3166-1", "is_in:country_code", "is_in_country",
+    "keys" : ["country_code", "ISO3166-1", "is_in:country_code", "is_in:country",
               "addr:country", "addr:country", "addr:country_code"],
     "values" : {
         "" : "country"

settings/import-full.style

@@ -186,7 +186,7 @@
     }
 },
 {
-    "keys" : ["country_code", "ISO3166-1", "is_in:country_code", "is_in_country",
+    "keys" : ["country_code", "ISO3166-1", "is_in:country_code", "is_in:country",
               "addr:country", "addr:country", "addr:country_code"],
     "values" : {
         "" : "country"

settings/import-street.style

@@ -42,7 +42,7 @@
     }
 },
 {
-    "keys" : ["country_code", "ISO3166-1", "is_in:country_code", "is_in_country",
+    "keys" : ["country_code", "ISO3166-1", "is_in:country_code", "is_in:country",
               "addr:country", "addr:country", "addr:country_code"],
     "values" : {
         "" : "country"

sql/functions.sql

@@ -547,7 +547,7 @@ BEGIN
 -- RAISE WARNING 'get_country_code, start: %', ST_AsText(place_centre);
 
   -- Try for a OSM polygon
-  FOR nearcountry IN select country_code from location_area_country where country_code is not null and not isguess and st_covers(geometry, place_centre) limit 1
+  FOR nearcountry IN select country_code from location_area_country where country_code is not null and st_covers(geometry, place_centre) limit 1
   LOOP
     RETURN nearcountry.country_code;
   END LOOP;
@@ -768,6 +768,28 @@ END;
 $$
 LANGUAGE plpgsql;
 
+CREATE OR REPLACE FUNCTION osmline_reinsert(node_id BIGINT, geom GEOMETRY)
+  RETURNS BOOLEAN
+  AS $$
+DECLARE
+  existingline RECORD;
+BEGIN
+   SELECT w.id FROM planet_osm_ways w, location_property_osmline p
+     WHERE p.linegeo && geom and p.osm_id = w.id and p.indexed_status = 0
+           and node_id = any(w.nodes) INTO existingline;
+
+   IF existingline.id is not NULL THEN
+       DELETE FROM location_property_osmline WHERE osm_id = existingline.id;
+       INSERT INTO location_property_osmline (osm_id, address, linegeo)
+         SELECT osm_id, address, geometry FROM place
+           WHERE osm_type = 'W' and osm_id = existingline.id;
+   END IF;
+
+   RETURN true;
+END;
+$$
+LANGUAGE plpgsql;
+
 
 CREATE OR REPLACE FUNCTION osmline_insert() RETURNS TRIGGER
   AS $$
@@ -887,11 +909,7 @@ BEGIN
     END IF;
 
     -- some postcorrections
-    IF NEW.class = 'place' THEN
-      IF NEW.type in ('continent', 'sea', 'country', 'state') AND NEW.osm_type = 'N' THEN
-        NEW.rank_address := 0;
-      END IF;
-    ELSEIF NEW.class = 'waterway' AND NEW.osm_type = 'R' THEN
+    IF NEW.class = 'waterway' AND NEW.osm_type = 'R' THEN
         -- Slightly promote waterway relations so that they are processed
         -- before their members.
         NEW.rank_search := NEW.rank_search - 1;
@@ -908,16 +926,14 @@ BEGIN
     NEW.country_code := NULL;
   END IF;
 
--- Block import below rank 22
---  IF NEW.rank_search > 22 THEN
---    RETURN NULL;
---  END IF;
-
   --DEBUG: RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
 
   RETURN NEW; -- %DIFFUPDATES% The following is not needed until doing diff updates, and slows the main index process down
 
-  IF NEW.rank_address > 0 THEN
+  IF NEW.osm_type = 'N' and NEW.rank_search > 28 THEN
+      -- might be part of an interpolation
+      result := osmline_reinsert(NEW.osm_id, NEW.geometry);
+  ELSEIF NEW.rank_address > 0 THEN
     IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
       -- Performance: We just can't handle re-indexing for country level changes
       IF st_area(NEW.geometry) < 1 THEN
@@ -1162,6 +1178,7 @@ TRIGGER
 DECLARE
 
   place_centroid GEOMETRY;
+  near_centroid GEOMETRY;
 
   search_maxdistance FLOAT[];
   search_mindistance FLOAT[];
@@ -1238,6 +1255,8 @@ BEGIN
   END IF;
 
   --DEBUG: RAISE WARNING 'Copy over address tags';
+  -- housenumber is a computed field, so start with an empty value
+  NEW.housenumber := NULL;
   IF NEW.address is not NULL THEN
       IF NEW.address ? 'conscriptionnumber' THEN
         i := getorcreate_housenumber_id(make_standard_name(NEW.address->'conscriptionnumber'));
@@ -1266,6 +1285,8 @@ BEGIN
   -- Speed up searches - just use the centroid of the feature
   -- cheaper but less acurate
   place_centroid := ST_PointOnSurface(NEW.geometry);
+  -- For searching near features rather use the centroid
+  near_centroid := ST_Envelope(NEW.geometry);
   NEW.centroid := null;
   NEW.postcode := null;
   --DEBUG: RAISE WARNING 'Computing preliminary centroid at %',ST_AsText(place_centroid);
@@ -1396,7 +1417,7 @@ BEGIN
     IF NEW.parent_place_id IS NULL AND addr_street IS NOT NULL THEN
       address_street_word_ids := get_name_ids(make_standard_name(addr_street));
       IF address_street_word_ids IS NOT NULL THEN
-        SELECT place_id from getNearestNamedRoadFeature(NEW.partition, place_centroid, address_street_word_ids) INTO NEW.parent_place_id;
+        SELECT place_id from getNearestNamedRoadFeature(NEW.partition, near_centroid, address_street_word_ids) INTO NEW.parent_place_id;
       END IF;
     END IF;
     --DEBUG: RAISE WARNING 'Checked for addr:street (%)', NEW.parent_place_id;
@@ -1404,7 +1425,7 @@ BEGIN
     IF NEW.parent_place_id IS NULL AND addr_place IS NOT NULL THEN
       address_street_word_ids := get_name_ids(make_standard_name(addr_place));
       IF address_street_word_ids IS NOT NULL THEN
-        SELECT place_id from getNearestNamedPlaceFeature(NEW.partition, place_centroid, address_street_word_ids) INTO NEW.parent_place_id;
+        SELECT place_id from getNearestNamedPlaceFeature(NEW.partition, near_centroid, address_street_word_ids) INTO NEW.parent_place_id;
       END IF;
     END IF;
     --DEBUG: RAISE WARNING 'Checked for addr:place (%)', NEW.parent_place_id;
@@ -1439,7 +1460,7 @@ BEGIN
           IF location.address ? 'street' THEN
             address_street_word_ids := get_name_ids(make_standard_name(location.address->'street'));
             IF address_street_word_ids IS NOT NULL THEN
-              SELECT place_id from getNearestNamedRoadFeature(NEW.partition, place_centroid, address_street_word_ids) INTO NEW.parent_place_id;
+              SELECT place_id from getNearestNamedRoadFeature(NEW.partition, near_centroid, address_street_word_ids) INTO NEW.parent_place_id;
               EXIT WHEN NEW.parent_place_id is not NULL;
             END IF;
           END IF;
@@ -1448,7 +1469,7 @@ BEGIN
           IF location.address ? 'place' THEN
             address_street_word_ids := get_name_ids(make_standard_name(location.address->'place'));
             IF address_street_word_ids IS NOT NULL THEN
-              SELECT place_id from getNearestNamedPlaceFeature(NEW.partition, place_centroid, address_street_word_ids) INTO NEW.parent_place_id;
+              SELECT place_id from getNearestNamedPlaceFeature(NEW.partition, near_centroid, address_street_word_ids) INTO NEW.parent_place_id;
               EXIT WHEN NEW.parent_place_id is not NULL;
             END IF;
           END IF;
@@ -1477,7 +1498,7 @@ BEGIN
 
     -- Still nothing, just use the nearest road
     IF NEW.parent_place_id IS NULL THEN
-      SELECT place_id FROM getNearestRoadFeature(NEW.partition, place_centroid) INTO NEW.parent_place_id;
+      SELECT place_id FROM getNearestRoadFeature(NEW.partition, near_centroid) INTO NEW.parent_place_id;
     END IF;
     --DEBUG: RAISE WARNING 'Checked for nearest way (%)', NEW.parent_place_id;
 
@@ -1500,7 +1521,7 @@ BEGIN
              NEW.postcode := location.postcode;
           END IF;
           IF NEW.postcode is null THEN
-            NEW.postcode := get_nearest_postcode(NEW.country_code, place_centroid);
+            NEW.postcode := get_nearest_postcode(NEW.country_code, NEW.geometry);
           END IF;
       END IF;
 
@@ -1803,7 +1824,7 @@ BEGIN
 
       -- RAISE WARNING '% isaddress: %', location.place_id, location_isaddress;
       -- Add it to the list of search terms
-      IF NOT %REVERSE-ONLY% AND location.rank_search > 4 THEN
+      IF NOT %REVERSE-ONLY% THEN
           nameaddress_vector := array_merge(nameaddress_vector, location.keywords::integer[]);
       END IF;
       INSERT INTO place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address)
@@ -2204,12 +2225,13 @@ BEGIN
         indexed_status = 2,
         geometry = NEW.geometry
         where place_id = existingplacex.place_id;
-
       -- if a node(=>house), which is part of a interpolation line, changes (e.g. the street attribute) => mark this line for reparenting 
       -- (already here, because interpolation lines are reindexed before nodes, so in the second call it would be too late)
-      IF NEW.osm_type='N' and NEW.class='place' and NEW.type='house' THEN
-          -- Is this node part of an interpolation line? search for it in location_property_osmline and mark the interpolation line for reparenting
-          update location_property_osmline p set indexed_status = 2 from planet_osm_ways w where p.linegeo && NEW.geometry and p.osm_id = w.id and NEW.osm_id = any(w.nodes);
+      IF NEW.osm_type='N'
+         and (coalesce(existing.address, ''::hstore) != coalesce(NEW.address, ''::hstore)
+             or existing.geometry::text != NEW.geometry::text)
+      THEN
+          result:= osmline_reinsert(NEW.osm_id, NEW.geometry);
       END IF;
 
       -- linked places should get potential new naming and addresses
@@ -2364,7 +2386,7 @@ BEGIN
 
   -- postcode table
   IF for_place_id IS NULL THEN
-    SELECT parent_place_id, country_code, rank_address, postcode, 'place', 'postcode'
+    SELECT parent_place_id, country_code, rank_search, postcode, 'place', 'postcode'
       FROM location_postcode
       WHERE place_id = in_place_id
       INTO for_place_id, searchcountrycode, searchrankaddress, searchpostcode,

sql/indices.src.sql

@@ -1,27 +1,27 @@
 -- Indices used only during search and update.
 -- These indices are created only after the indexing process is done.
 
-CREATE INDEX idx_word_word_id on word USING BTREE (word_id) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_word_word_id on word USING BTREE (word_id) {ts:search-index};
 
-CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id) {ts:search-index};
 
-DROP INDEX IF EXISTS idx_placex_rank_search;
-CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search) {ts:search-index};
-CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address) {ts:search-index};
-CREATE INDEX idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) {ts:address-index} where indexed_status > 0;
-CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) {ts:search-index} where parent_place_id IS NOT NULL;
+DROP INDEX CONCURRENTLY IF EXISTS idx_placex_rank_search;
+CREATE INDEX CONCURRENTLY idx_placex_rank_search ON placex USING BTREE (rank_search) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_placex_rank_address ON placex USING BTREE (rank_address) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) {ts:address-index} where indexed_status > 0;
+CREATE INDEX CONCURRENTLY idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) {ts:search-index} where parent_place_id IS NOT NULL;
 
-CREATE INDEX idx_placex_geometry_reverse_lookupPoint
+CREATE INDEX CONCURRENTLY idx_placex_geometry_reverse_lookupPoint
   ON placex USING gist (geometry) {ts:search-index}
   WHERE (name is not null or housenumber is not null or rank_address between 26 and 27)
     AND class not in ('railway','tunnel','bridge','man_made')
     AND rank_address >= 26 AND indexed_status = 0 AND linked_place_id is null;
-CREATE INDEX idx_placex_geometry_reverse_lookupPolygon
+CREATE INDEX CONCURRENTLY idx_placex_geometry_reverse_lookupPolygon
   ON placex USING gist (geometry) {ts:search-index}
   WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')
     AND rank_address between 4 and 25 AND type != 'postcode'
     AND name is not null AND indexed_status = 0 AND linked_place_id is null;
-CREATE INDEX idx_placex_geometry_reverse_placeNode
+CREATE INDEX CONCURRENTLY idx_placex_geometry_reverse_placeNode
   ON placex USING gist (geometry) {ts:search-index}
   WHERE osm_type = 'N' AND rank_search between 5 and 25
     AND class = 'place' AND type != 'postcode'
@@ -29,14 +29,14 @@ CREATE INDEX idx_placex_geometry_reverse_placeNode
 
 GRANT SELECT ON table country_osm_grid to "{www-user}";
 
-CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id) {ts:address-index};
+CREATE INDEX CONCURRENTLY idx_location_area_country_place_id ON location_area_country USING BTREE (place_id) {ts:address-index};
 
-CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {ts:search-index};
-CREATE INDEX idx_osmline_parent_osm_id ON location_property_osmline USING BTREE (osm_id) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_osmline_parent_osm_id ON location_property_osmline USING BTREE (osm_id) {ts:search-index};
 
-DROP INDEX IF EXISTS place_id_idx;
-CREATE UNIQUE INDEX idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index};
+DROP INDEX CONCURRENTLY IF EXISTS place_id_idx;
+CREATE UNIQUE INDEX CONCURRENTLY idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index};
 
 
-CREATE UNIQUE INDEX idx_postcode_id ON location_postcode USING BTREE (place_id) {ts:search-index};
-CREATE INDEX idx_postcode_postcode ON location_postcode USING BTREE (postcode) {ts:search-index};
+CREATE UNIQUE INDEX CONCURRENTLY idx_postcode_id ON location_postcode USING BTREE (place_id) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_postcode_postcode ON location_postcode USING BTREE (postcode) {ts:search-index};

sql/indices_search.src.sql

@@ -1,6 +1,6 @@
 -- Indices used for /search API.
 -- These indices are created only after the indexing process is done.
 
-CREATE INDEX idx_search_name_nameaddress_vector ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {ts:search-index};
-CREATE INDEX idx_search_name_name_vector ON search_name USING GIN (name_vector) WITH (fastupdate = off) {ts:search-index};
-CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_search_name_nameaddress_vector ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_search_name_name_vector ON search_name USING GIN (name_vector) WITH (fastupdate = off) {ts:search-index};
+CREATE INDEX CONCURRENTLY idx_search_name_centroid ON search_name USING GIST (centroid) {ts:search-index};

sql/partition-functions.src.sql

@@ -6,11 +6,9 @@ BEGIN
 -- start
   IF in_partition = -partition- THEN
     FOR r IN 
-      SELECT place_id, keywords, rank_address, rank_search, min(ST_Distance(feature, centroid)) as distance, isguess, postcode, centroid FROM (
-        SELECT * FROM location_area_large_-partition- WHERE ST_Intersects(geometry, feature) and rank_search < maxrank
-        UNION ALL
-        SELECT * FROM location_area_country WHERE ST_Intersects(geometry, feature) and rank_search < maxrank
-      ) as location_area
+      SELECT place_id, keywords, rank_address, rank_search, min(ST_Distance(feature, centroid)) as distance, isguess, postcode, centroid
+      FROM location_area_large_-partition-
+      WHERE ST_Intersects(geometry, feature) and rank_search < maxrank
       GROUP BY place_id, keywords, rank_address, rank_search, isguess, postcode, centroid
       ORDER BY rank_address, isin_tokens && keywords desc, isguess asc,
         ST_Distance(feature, centroid) *
@@ -64,9 +62,9 @@ BEGIN
     RETURN TRUE;
   END IF;
 
-  IF in_rank_search <= 4 THEN
-    INSERT INTO location_area_country (partition, place_id, country_code, keywords, rank_search, rank_address, isguess, centroid, geometry)
-      values (in_partition, in_place_id, in_country_code, in_keywords, in_rank_search, in_rank_address, in_estimate, in_centroid, in_geometry);
+  IF in_rank_search <= 4 and not in_estimate THEN
+    INSERT INTO location_area_country (place_id, country_code, geometry)
+      values (in_place_id, in_country_code, in_geometry);
     RETURN TRUE;
   END IF;
 
@@ -97,7 +95,7 @@ BEGIN
           ST_Distance(centroid, point) as distance, null as isguess
           FROM search_name_-partition-
           WHERE name_vector && isin_token
-          AND ST_DWithin(centroid, point, 0.015)
+          AND centroid && ST_Expand(point, 0.015)
           AND search_rank between 26 and 27
       ORDER BY distance ASC limit 1
     LOOP
@@ -125,7 +123,7 @@ BEGIN
           ST_Distance(centroid, point) as distance, null as isguess
           FROM search_name_-partition-
           WHERE name_vector && isin_token
-          AND ST_DWithin(centroid, point, 0.04)
+          AND centroid && ST_Expand(point, 0.04)
           AND search_rank between 16 and 22
       ORDER BY distance ASC limit 1
     LOOP

sql/partition-tables.src.sql

@@ -35,9 +35,6 @@ CREATE TABLE search_name_blank (
   );
 
 
-CREATE TABLE location_area_country () INHERITS (location_area_large) {ts:address-data};
-CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {ts:address-index};
-
 -- start
 CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large) {ts:address-data};
 CREATE INDEX idx_location_area_large_-partition-_place_id ON location_area_large_-partition- USING BTREE (place_id) {ts:address-index};
@@ -45,8 +42,8 @@ CREATE INDEX idx_location_area_large_-partition-_geometry ON location_area_large
 
 CREATE TABLE search_name_-partition- () INHERITS (search_name_blank) {ts:address-data};
 CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id) {ts:address-index};
-CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid) {ts:address-index};
-CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off) {ts:address-index};
+CREATE INDEX idx_search_name_-partition-_centroid_street ON search_name_-partition- USING GIST (centroid) {ts:address-index} where search_rank between 26 and 27;
+CREATE INDEX idx_search_name_-partition-_centroid_place ON search_name_-partition- USING GIST (centroid) {ts:address-index} where search_rank between 2 and 25;
 
 DROP TABLE IF EXISTS location_road_-partition-;
 CREATE TABLE location_road_-partition- (

sql/tables.sql

@@ -36,6 +36,7 @@ GRANT SELECT ON new_query_log TO "{www-user}" ;
 
 GRANT SELECT ON TABLE country_name TO "{www-user}";
 GRANT SELECT ON TABLE gb_postcode TO "{www-user}";
+GRANT SELECT ON TABLE us_postcode TO "{www-user}";
 
 drop table IF EXISTS word;
 CREATE TABLE word (
@@ -69,6 +70,15 @@ CREATE TABLE location_area (
 
 CREATE TABLE location_area_large () INHERITS (location_area);
 
+DROP TABLE IF EXISTS location_area_country;
+CREATE TABLE location_area_country (
+  place_id BIGINT,
+  country_code varchar(2),
+  geometry GEOMETRY(Geometry, 4326)
+  ) {ts:address-data};
+CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {ts:address-index};
+
+
 drop table IF EXISTS location_property CASCADE;
 CREATE TABLE location_property (
   place_id BIGINT,

sql/tiger_import_start.sql

@@ -26,6 +26,11 @@ BEGIN
     endnumber = in_startnumber;
   END IF;
 
+  IF startnumber < 0 THEN
+    RAISE WARNING 'Negative house number range (% to %) on %, %', startnumber, endnumber, in_street, in_isin;
+    RETURN 0;
+  END IF;
+
   numberrange := endnumber - startnumber;
 
   IF (interpolationtype = 'odd' AND startnumber%2 = 0) OR (interpolationtype = 'even' AND startnumber%2 = 1) THEN

sql/update-postcodes.sql

@@ -6,7 +6,7 @@ SELECT country_code,
        ST_Centroid(ST_Collect(ST_Centroid(geometry))) as centroid
   FROM placex
  WHERE address ? 'postcode'
-       AND address->'postcode' NOT SIMILAR TO '%(,|;)%'
+       AND address->'postcode' NOT SIMILAR TO '%(,|;|:)%'
        AND geometry IS NOT null
 GROUP BY country_code, pc;
 

test/bdd/api/lookup/simple.feature

@@ -4,15 +4,30 @@ Feature: Places by osm_type and osm_id Tests
 
     Scenario Outline: address lookup for existing node, way, relation
         When sending <format> lookup query for N3284625766,W6065798,,R123924,X99,N0
-        Then the result is valid <format>
+        Then the result is valid <outformat>
         And exactly 3 results are returned
 
     Examples:
-        | format  |
-        | xml     |
-        | json    |
-        | geojson |
+        | format      | outformat   |
+        | xml         | xml         |
+        | json        | json        |
+        | jsonv2      | json        |
+        | geojson     | geojson     |
+        | geocodejson | geocodejson |
 
     Scenario: address lookup for non-existing or invalid node, way, relation
         When sending xml lookup query for X99,,N0,nN158845944,ABC,,W9
         Then exactly 0 results are returned
+
+    Scenario Outline: Boundingbox is returned
+        When sending <format> lookup query for N3284625766,W6065798
+        Then exactly 2 results are returned
+        And result 0 has bounding box in -32.812,-32.811,-56.509,-56.508
+        And result 1 has bounding box in 47.14,47.15,9.51,9.53
+
+    Examples:
+      | format |
+      | json |
+      | jsonv2 |
+      | geojson |
+      | xml |

test/bdd/db/import/parenting.feature

@@ -224,7 +224,7 @@ Feature: Parenting of objects
         When importing
         Then placex contains
          | object | parent_place_id |
-         | W1     | W3 |
+         | W1     | W2 |
 
     Scenario: Building with addr:street tags
         Given the scene building-on-street-corner

test/bdd/db/import/placex.feature

@@ -220,8 +220,8 @@ Feature: Import into placex
           | R21    | 30          | 30 |
           | R22    | 30          | 30 |
           | R23    | 30          | 30 |
-          | R40    | 4           | 4 |
-          | R41    | 8           | 8 |
+          | R40    | 4           | 0 |
+          | R41    | 8           | 0 |
 
     Scenario: search and address ranks for highways correctly assigned
         Given the scene roads-with-pois

test/bdd/db/update/interpolation.feature

@@ -275,7 +275,6 @@ Feature: Update of address interpolations
           | parent_place_id | start | end |
           | W2              | 2     | 6 |
 
-    @Fail
     Scenario: housenumber added in middle of interpolation
       Given the grid
           | 1 |  |  |   |  | 2 |
@@ -334,7 +333,6 @@ Feature: Update of address interpolations
           | parent_place_id | start | end | geometry |
           | W1              | 2     | 10  | 3,4,5    |
 
-    @Fail
     Scenario: Change the start housenumber
       Given the grid
           | 1 |  | 2 |

test/bdd/db/update/parenting.feature

@@ -0,0 +1,38 @@
+@DB
+Feature: Update parenting of objects
+
+Scenario: POI inside building inherits addr:street change
+        Given the scene building-on-street-corner
+        And the named places
+         | osm | class   | type       | geometry |
+         | N1  | amenity | bank       | :n-inner |
+         | N2  | shop    | bakery     | :n-edge-NS |
+         | N3  | shop    | supermarket| :n-edge-WE |
+        And the places
+         | osm | class    | type | addr_place | housenr | geometry |
+         | W1  | building | yes  | nowhere    | 3       | :w-building |
+        And the places
+         | osm | class    | type        | name | geometry |
+         | W2  | highway  | primary     | bar  | :w-WE |
+         | W3  | highway  | residential | foo  | :w-NS |
+        When importing
+        Then placex contains
+         | object | parent_place_id | housenumber |
+         | W1     | W2              | 3 |
+         | N1     | W3              | 3 |
+         | N2     | W3              | 3 |
+         | N3     | W2              | 3 |
+        When updating places
+         | osm | class    | type | street | addr_place | housenr | geometry    |
+         | W1  | building | yes  | foo    | nowhere    | 3       | :w-building |
+        And updating places
+         | osm | class   | type       | name | geometry |
+         | N3  | shop    | supermarket| well | :n-edge-WE |
+        Then placex contains
+         | object | parent_place_id | housenumber |
+         | W1     | W3              | 3 |
+         | N1     | W3              | 3 |
+         | N2     | W3              | 3 |
+         | N3     | W3              | 3 |
+
+

test/bdd/environment.py

@@ -195,6 +195,7 @@ class NominatimEnvironment(object):
         proc = subprocess.Popen(cmd, cwd=self.build_dir,
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         (outp, outerr) = proc.communicate()
+        outerr = outerr.decode('utf-8').replace('\\n', '\n')
         logger.debug("run_nominatim_script: %s\n%s\n%s" % (cmd, outp, outerr))
         assert (proc.returncode == 0), "Script '%s' failed:\n%s\n%s\n" % (script, outp, outerr)
 

test/bdd/osm2pgsql/import/tags.feature

@@ -96,9 +96,9 @@ Feature: Tag evaluation
         Then place contains
          | object | name |
          | N1     | 'name: de' : 'Foo', 'name' : 'real1' |
-         | N2     | 'name: de' : 'Foo', 'name' : 'real2' |
-         | N3     | 'name: de' : 'Foo', 'name:\\\\' : 'real3' |
-         | N4     | 'name: de' : 'Foo', 'name' : 'rea\\l3' |
+         | N2     | 'name:\nde' : 'Foo', 'name' : 'real2' |
+         | N3     | 'name:\tde' : 'Foo', 'name:\\\\' : 'real3' |
+         | N4     | 'name:\tde' : 'Foo', 'name' : 'rea\\l3' |
 
     Scenario: Unprintable character in address tag are maintained
         When loading osm data

test/bdd/osm2pgsql/update/simple.feature

@@ -24,3 +24,38 @@ Feature: Update of simple objects by osm2pgsql
           | object     | class   | type       | name |
           | N1:tourism | tourism | hotel      | 'name' : 'foo' |
 
+    Scenario: Downgrading a highway to one that is dropped without name
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=residential Nn100,n101
+          """
+        Then place contains
+          | object     |
+          | W1:highway |
+        When updating osm data
+          """
+          w1 Thighway=service Nn100,n101
+          """
+        Then place has no entry for W1
+
+    Scenario: Downgrading a highway when a second tag is present
+        When loading osm data
+          """
+          n100 x0 y0
+          n101 x0.0001 y0.0001
+          w1 Thighway=residential,tourism=hotel Nn100,n101
+          """
+        Then place contains
+          | object     |
+          | W1:highway |
+          | W1:tourism |
+        When updating osm data
+          """
+          w1 Thighway=service,tourism=hotel Nn100,n101
+          """
+        Then place has no entry for W1:highway
+        And place contains
+          | object     |
+          | W1:tourism |

test/bdd/steps/queries.py

@@ -455,8 +455,12 @@ def website_lookup_request(context, fmt, query):
 
     if fmt == 'json ':
         outfmt = 'json'
+    elif fmt == 'jsonv2 ':
+        outfmt = 'json'
     elif fmt == 'geojson ':
         outfmt = 'geojson'
+    elif fmt == 'geocodejson ':
+        outfmt = 'geocodejson'
     else:
         outfmt = 'xml'
 

test/php/Nominatim/PhraseTest.php

@@ -4,6 +4,29 @@ namespace Nominatim;
 
 require_once(CONST_BasePath.'/lib/Phrase.php');
 
+class TokensFullSet
+{
+    public function containsAny($sTerm)
+    {
+        return true;
+    }
+}
+
+// phpcs:ignore PSR1.Classes.ClassDeclaration.MultipleClasses
+class TokensPartialSet
+{
+    public function __construct($aTokens)
+    {
+        $this->aTokens = array_flip($aTokens);
+    }
+
+    public function containsAny($sTerm)
+    {
+        return isset($this->aTokens[$sTerm]);
+    }
+}
+
+// phpcs:ignore PSR1.Classes.ClassDeclaration.MultipleClasses
 class PhraseTest extends \PHPUnit\Framework\TestCase
 {
 
@@ -21,6 +44,7 @@ class PhraseTest extends \PHPUnit\Framework\TestCase
     public function testEmptyPhrase()
     {
         $oPhrase = new Phrase('', '');
+        $oPhrase->computeWordSets(new TokensFullSet());
 
         $this->assertEquals(
             array(array('')),
@@ -32,6 +56,7 @@ class PhraseTest extends \PHPUnit\Framework\TestCase
     public function testSingleWordPhrase()
     {
         $oPhrase = new Phrase('a', '');
+        $oPhrase->computeWordSets(new TokensFullSet());
 
         $this->assertEquals(
             '(a)',
@@ -43,20 +68,23 @@ class PhraseTest extends \PHPUnit\Framework\TestCase
     public function testMultiWordPhrase()
     {
         $oPhrase = new Phrase('a b', '');
+        $oPhrase->computeWordSets(new TokensFullSet());
         $this->assertEquals(
             '(a b),(a|b)',
             $this->serializeSets($oPhrase->getWordSets())
         );
 
         $oPhrase = new Phrase('a b c', '');
+        $oPhrase->computeWordSets(new TokensFullSet());
         $this->assertEquals(
-            '(a b c),(a|b c),(a|b|c),(a b|c)',
+            '(a b c),(a|b c),(a b|c),(a|b|c)',
             $this->serializeSets($oPhrase->getWordSets())
         );
 
         $oPhrase = new Phrase('a b c d', '');
+        $oPhrase->computeWordSets(new TokensFullSet());
         $this->assertEquals(
-            '(a b c d),(a|b c d),(a|b|c d),(a|b|c|d),(a|b c|d),(a b|c d),(a b|c|d),(a b c|d)',
+            '(a b c d),(a b c|d),(a b|c d),(a|b c d),(a b|c|d),(a|b c|d),(a|b|c d),(a|b|c|d)',
             $this->serializeSets($oPhrase->getWordSets())
         );
     }
@@ -65,25 +93,47 @@ class PhraseTest extends \PHPUnit\Framework\TestCase
     public function testInverseWordSets()
     {
         $oPhrase = new Phrase('a b c', '');
+        $oPhrase->computeWordSets(new TokensFullSet());
         $oPhrase->invertWordSets();
 
         $this->assertEquals(
-            '(a b c),(c|a b),(c|b|a),(b c|a)',
+            '(a b c),(b c|a),(c|a b),(c|b|a)',
             $this->serializeSets($oPhrase->getWordSets())
         );
     }
 
 
-    public function testMaxDepth()
+    public function testMaxWordSets()
     {
         $oPhrase = new Phrase(join(' ', array_fill(0, 4, 'a')), '');
+        $oPhrase->computeWordSets(new TokensFullSet());
         $this->assertEquals(8, count($oPhrase->getWordSets()));
         $oPhrase->invertWordSets();
         $this->assertEquals(8, count($oPhrase->getWordSets()));
 
         $oPhrase = new Phrase(join(' ', array_fill(0, 18, 'a')), '');
-        $this->assertEquals(41226, count($oPhrase->getWordSets()));
+        $oPhrase->computeWordSets(new TokensFullSet());
+        $this->assertEquals(100, count($oPhrase->getWordSets()));
         $oPhrase->invertWordSets();
-        $this->assertEquals(41226, count($oPhrase->getWordSets()));
+        $this->assertEquals(100, count($oPhrase->getWordSets()));
+    }
+
+
+    public function testPartialTokensShortTerm()
+    {
+        $oPhrase = new Phrase('a b c d', '');
+        $oPhrase->computeWordSets(new TokensPartialSet(array('a', 'b', 'd', 'b c', 'b c d')));
+        $this->assertEquals(
+            '(a|b c d),(a|b c|d)',
+            $this->serializeSets($oPhrase->getWordSets())
+        );
+    }
+
+
+    public function testPartialTokensLongTerm()
+    {
+        $oPhrase = new Phrase(join(' ', array_fill(0, 18, 'a')), '');
+        $oPhrase->computeWordSets(new TokensPartialSet(array('a', 'a a a a a')));
+        $this->assertEquals(80, count($oPhrase->getWordSets()));
     }
 }

utils/check_server_for_updates.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import sys
 from osmium.replication import server

utils/import_wikipedia.sh

@@ -1,53 +0,0 @@
-#!/bin/bash
-
-psqlcmd=psql wikipedia2013
-mysql2pgsqlcmd=./mysql2pgsql.perl /dev/stdin /dev/stdout
-
-language=( "ar" "bg" "ca" "cs" "da" "de" "en" "es" "eo" "eu" "fa" "fr" "ko" "hi" "hr" "id" "it" "he" "lt" "hu" "ms" "nl" "ja" "no" "pl" "pt" "kk" "ro" "ru" "sk" "sl" "sr" "fi" "sv" "tr" "uk" "vi" "vo" "war" "zh" )
-
-# wikipedia pages and links
-echo "CREATE TABLE linkcounts (language text, title text, count integer, sumcount integer, lat double, lon double );"  | $psqlcmd
-echo "CREATE TABLE wikipedia_redirect (language text, from_title text, to_title text );"  | $psqlcmd
-
-for i in "${language[@]}"
-do
-    wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-page.sql.gz
-    wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-pagelinks.sql.gz
-    wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-langlinks.sql.gz
-    wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-redirect.sql.gz
-done
-
-for i in "${language[@]}"
-do
-    gzip -dc ${i}wiki-latest-pagelinks.sql.gz | sed "s/\`pagelinks\`/\`${i}pagelinks\`/g" | $mysql2pgsqlcmd | $psqlcmd
-    gzip -dc ${i}wiki-latest-page.sql.gz | sed "s/\`page\`/\`${i}page\`/g" | $mysql2pgsqlcmd | $psqlcmd
-    gzip -dc ${i}wiki-latest-langlinks.sql.gz | sed "s/\`langlinks\`/\`${i}langlinks\`/g" | $mysql2pgsqlcmd | $psqlcmd
-    gzip -dc ${i}wiki-latest-redirect.sql.gz | sed "s/\`redirect\`/\`${i}redirect\`/g" | $mysql2pgsqlcmd | $psqlcmd
-done
-
-for i in "${language[@]}"
-do
-    echo "create table ${i}pagelinkcount as select pl_title as title,count(*) as count from ${i}pagelinks where pl_namespace = 0 group by pl_title;" | $psqlcmd
-    echo "insert into linkcounts select '${i}',pl_title,count(*) from ${i}pagelinks where pl_namespace = 0 group by pl_title;" | $psqlcmd
-    echo "insert into wikipedia_redirect select '${i}',page_title,rd_title from ${i}redirect join ${i}page on (rd_from = page_id) where page_namespace = 0 and rd_namespace = 0;" | $psqlcmd
-    echo "alter table ${i}pagelinkcount add column othercount integer;" | $psqlcmd
-    echo "update ${i}pagelinkcount set othercount = 0;" | $psqlcmd
-    for j in "${language[@]}"
-    do
-        echo "update ${i}pagelinkcount set othercount = ${i}pagelinkcount.othercount + x.count from (select page_title as title,count from ${i}langlinks join ${i}page on (ll_from = page_id) join ${j}pagelinkcount on (ll_lang = '${j}' and ll_title = title)) as x where x.title = ${i}pagelinkcount.title;" | $psqlcmd
-    done
-    echo "insert into wikipedia_article select '${i}', title, count, othercount, count+othercount from ${i}pagelinkcount;" | $psqlcmd
-done
-
-echo "update wikipedia_article set importance = log(totalcount)/log((select max(totalcount) from wikipedia_article))" | $psqlcmd
-
-# precalculated lat,lon from dbpedia
-wget http://downloads.dbpedia.org/current/en/geo_coordinates_en.nq.bz2
-bzip2 -dc geo_coordinates_en.nq.bz2 | grep http://www.georss.org/georss/point | sed 's|<http://dbpedia.org/resource/[^>]*> *<http://www.georss.org/georss/point> "\(-\?[-0-9.E]\+\) \(-\?[-0-9.E]\+\)"@en <http://\([a-z][a-z]\).wikipedia.org/wiki/\([^#]\+\)#> .|update pagelinks set lat=\1, lon=\2 where language = '"'"'\3'"'"' and title = decode_url_part('"'"'\4'"'"');|g' | $psqlcmd
-
-# media wiki dumper
-wget https://github.com/bcollier/mwdumper/blob/master/build/mwdumper.jar
-
-# latest english wikipedia articles
-wget http://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2
-java -jar mwdumper.jar --format=sql:1.5 enwiki-latest-pages-articles.xml.bz2 | ./mysql2pgsql.perl /dev/stdin /dev/stdout | sed 's/"text (/text ("/g' | sed 's/"old_flags)"/"old_flags")/g' | sed 's/"revision (/revision ("/g' | sed 's/"rev_deleted)"/"rev_deleted")/g' | sed 's/"page (/page ("/g' | sed 's/"page_len)"/"page_len")/g' | sed "s/DATE_ADD(E'1970-01-01', INTERVAL UNIX_TIMESTAMP() SECOND)[+]//g" | sed 's/RAND()/0/g' | $psqlcmd

utils/osm_file_date.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import osmium
 import sys

utils/specialphrases.php

@@ -27,77 +27,127 @@ if ($aCMDResult['wiki-import']) {
     foreach (explode(',', $sLanguageIn) as $sLanguage) {
         $sURL = 'https://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Special_Phrases/'.strtoupper($sLanguage);
         $sWikiPageXML = file_get_contents($sURL);
-        if (preg_match_all('#\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([\\-YN])#', $sWikiPageXML, $aMatches, PREG_SET_ORDER)) {
-            foreach ($aMatches as $aMatch) {
-                $sLabel = trim($aMatch[1]);
-                if ($oNormalizer !== null) {
-                    $sTrans = pg_escape_string($oNormalizer->transliterate($sLabel));
-                } else {
-                    $sTrans = null;
-                }
-                $sClass = trim($aMatch[2]);
-                $sType = trim($aMatch[3]);
-                // hack around a bug where building=yes was imported with
-                // quotes into the wiki
-                $sType = preg_replace('/"/', '', $sType);
-                // sanity check, in case somebody added garbage in the wiki
-                if (preg_match('/^\\w+$/', $sClass) < 1
-                    || preg_match('/^\\w+$/', $sType) < 1
-                ) {
-                    trigger_error("Bad class/type for language $sLanguage: $sClass=$sType");
-                    exit;
-                }
-                // blacklisting: disallow certain class/type combinations
-                if (isset($aTagsBlacklist[$sClass]) && in_array($sType, $aTagsBlacklist[$sClass])) {
-                    // fwrite(STDERR, "Blacklisted: ".$sClass."/".$sType."\n");
-                    continue;
-                }
-                // whitelisting: if class is in whitelist, allow only tags in the list
-                if (isset($aTagsWhitelist[$sClass]) && !in_array($sType, $aTagsWhitelist[$sClass])) {
-                    // fwrite(STDERR, "Non-Whitelisted: ".$sClass."/".$sType."\n");
-                    continue;
-                }
-                $aPairs[$sClass.'|'.$sType] = array($sClass, $sType);
 
-                switch (trim($aMatch[4])) {
-                    case 'near':
-                        echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sTrans', '$sClass', '$sType', 'near');\n";
-                        break;
-                    case 'in':
-                        echo "select getorcreate_amenityoperator(make_standard_name('".pg_escape_string($sLabel)."'), '$sTrans', '$sClass', '$sType', 'in');\n";
-                        break;
-                    default:
-                        echo "select getorcreate_amenity(make_standard_name('".pg_escape_string($sLabel)."'), '$sTrans', '$sClass', '$sType');\n";
-                        break;
-                }
+        if (!preg_match_all(
+            '#\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([^|]+) \\|\\| ([\\-YN])#',
+            $sWikiPageXML,
+            $aMatches,
+            PREG_SET_ORDER
+        )) {
+            continue;
+        }
+
+        foreach ($aMatches as $aMatch) {
+            $sLabel = trim($aMatch[1]);
+            if ($oNormalizer !== null) {
+                $sTrans = pg_escape_string($oNormalizer->transliterate($sLabel));
+            } else {
+                $sTrans = null;
+            }
+            $sClass = trim($aMatch[2]);
+            $sType = trim($aMatch[3]);
+            // hack around a bug where building=yes was imported with
+            // quotes into the wiki
+            $sType = preg_replace('/(&quot;|")/', '', $sType);
+            // sanity check, in case somebody added garbage in the wiki
+            if (preg_match('/^\\w+$/', $sClass) < 1
+                || preg_match('/^\\w+$/', $sType) < 1
+            ) {
+                trigger_error("Bad class/type for language $sLanguage: $sClass=$sType");
+                exit;
+            }
+            // blacklisting: disallow certain class/type combinations
+            if (isset($aTagsBlacklist[$sClass]) && in_array($sType, $aTagsBlacklist[$sClass])) {
+                // fwrite(STDERR, "Blacklisted: ".$sClass."/".$sType."\n");
+                continue;
+            }
+            // whitelisting: if class is in whitelist, allow only tags in the list
+            if (isset($aTagsWhitelist[$sClass]) && !in_array($sType, $aTagsWhitelist[$sClass])) {
+                // fwrite(STDERR, "Non-Whitelisted: ".$sClass."/".$sType."\n");
+                continue;
+            }
+            $aPairs[$sClass.'|'.$sType] = array($sClass, $sType);
+
+            switch (trim($aMatch[4])) {
+                case 'near':
+                    printf(
+                        "SELECT getorcreate_amenityoperator(make_standard_name('%s'), '%s', '%s', '%s', 'near');\n",
+                        pg_escape_string($sLabel),
+                        $sTrans,
+                        $sClass,
+                        $sType
+                    );
+                    break;
+                case 'in':
+                    printf(
+                        "SELECT getorcreate_amenityoperator(make_standard_name('%s'), '%s', '%s', '%s', 'in');\n",
+                        pg_escape_string($sLabel),
+                        $sTrans,
+                        $sClass,
+                        $sType
+                    );
+                    break;
+                default:
+                    printf(
+                        "SELECT getorcreate_amenity(make_standard_name('%s'), '%s', '%s', '%s');\n",
+                        pg_escape_string($sLabel),
+                        $sTrans,
+                        $sClass,
+                        $sType
+                    );
+                    break;
             }
         }
     }
 
-    echo 'create index idx_placex_classtype on placex (class, type);';
+    echo 'CREATE INDEX idx_placex_classtype ON placex (class, type);';
 
     foreach ($aPairs as $aPair) {
-        echo 'create table place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]);
-        if (CONST_Tablespace_Aux_Data)
-            echo ' tablespace '.CONST_Tablespace_Aux_Data;
-        echo ' as select place_id as place_id,st_centroid(geometry) as centroid from placex where ';
-        echo "class = '".pg_escape_string($aPair[0])."' and type = '".pg_escape_string($aPair[1])."'";
-        echo ";\n";
+        $sql_tablespace = CONST_Tablespace_Aux_Data ? ' TABLESPACE '.CONST_Tablespace_Aux_Data : '';
 
-        echo 'CREATE INDEX idx_place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).'_centroid ';
-        echo 'ON place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).' USING GIST (centroid)';
-        if (CONST_Tablespace_Aux_Index)
-            echo ' tablespace '.CONST_Tablespace_Aux_Index;
-        echo ";\n";
+        printf(
+            'CREATE TABLE place_classtype_%s_%s'
+            . $sql_tablespace
+            . ' AS'
+            . ' SELECT place_id AS place_id,st_centroid(geometry) AS centroid FROM placex'
+            . " WHERE class = '%s' AND type = '%s'"
+            . ";\n",
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1]),
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1])
+        );
 
-        echo 'CREATE INDEX idx_place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).'_place_id ';
-        echo 'ON place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).' USING btree(place_id)';
-        if (CONST_Tablespace_Aux_Index)
-            echo ' tablespace '.CONST_Tablespace_Aux_Index;
-        echo ";\n";
+        printf(
+            'CREATE INDEX idx_place_classtype_%s_%s_centroid'
+            . ' ON place_classtype_%s_%s USING GIST (centroid)'
+            . $sql_tablespace
+            . ";\n",
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1]),
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1])
+        );
 
-        echo 'GRANT SELECT ON place_classtype_'.pg_escape_string($aPair[0]).'_'.pg_escape_string($aPair[1]).' TO "'.CONST_Database_Web_User."\";\n";
+        printf(
+            'CREATE INDEX idx_place_classtype_%s_%s_place_id'
+            . ' ON place_classtype_%s_%s USING btree(place_id)'
+            . $sql_tablespace
+            . ";\n",
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1]),
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1])
+        );
+
+        printf(
+            'GRANT SELECT ON place_classtype_%s_%s TO "%s"'
+            . ";\n",
+            pg_escape_string($aPair[0]),
+            pg_escape_string($aPair[1]),
+            CONST_Database_Web_User
+        );
     }
 
-    echo 'drop index idx_placex_classtype;';
+    echo 'DROP INDEX idx_placex_classtype;';
 }

vagrant/Install-on-Ubuntu-18-nginx.sh

@@ -97,7 +97,7 @@ server {
         if (!-f \$document_root\$fastcgi_script_name) {
             return 404;
         }
-        fastcgi_pass unix:/var/run/php7.2-fpm.sock;
+        fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
         fastcgi_index search.php;
         include fastcgi.conf;
     }

website/details.php

@@ -32,14 +32,12 @@ $sLanguagePrefArraySQL = $oDB->getArraySQL($oDB->getDBQuotedList($aLangPrefOrder
 
 if ($sOsmType && $iOsmId > 0) {
     $sSQL = 'SELECT place_id FROM placex WHERE osm_type = :type AND osm_id = :id';
-    $aSQLParams = array(':type' => $sOsmType, ':id' => $iOsmId);
     // osm_type and osm_id are not unique enough
     if ($sClass) {
-        $sSQL .= ' AND class= :class';
-        $aSQLParams[':class'] = $sClass;
+        $sSQL .= " AND class='".$sClass."'";
     }
     $sSQL .= ' ORDER BY class ASC';
-    $sPlaceId = $oDB->getOne($sSQL, $aSQLParams);
+    $sPlaceId = $oDB->getOne($sSQL, array(':type' => $sOsmType, ':id' => $iOsmId));
 
     // Be nice about our error messages for broken geometry
 

website/hierarchy.php

@@ -58,7 +58,7 @@ if (CONST_Use_Aux_Location_data) {
 }
 
 
-$oAddressLookup = new AddressDetails($oDB, $iPlaceID, -1, $aLangPrefOrder);
+$oAddressLookup = new Nominatim\AddressDetails($oDB, $iPlaceID, -1, $aLangPrefOrder);
 $aPlaceAddress = array_reverse($oAddressLookup->getAddressDetails());
 
 if (empty($aPlaceAddress)) userError('Unknown place id.');

website/lookup.php

@@ -9,7 +9,7 @@ ini_set('memory_limit', '200M');
 $oParams = new Nominatim\ParameterParser();
 
 // Format for output
-$sOutputFormat = $oParams->getSet('format', array('xml', 'json', 'geojson'), 'xml');
+$sOutputFormat = $oParams->getSet('format', array('xml', 'json', 'jsonv2', 'geojson', 'geocodejson'), 'xml');
 set_exception_handler_by_format($sOutputFormat);
 
 // Preferred language
@@ -49,8 +49,22 @@ foreach ($aOsmIds as $sItem) {
             $oResult = $oPlace;
             unset($oResult['aAddress']);
             if (isset($oPlace['aAddress'])) $oResult['address'] = $oPlace['aAddress'];
-            unset($oResult['langaddress']);
-            $oResult['name'] = $oPlace['langaddress'];
+            if ($sOutputFormat != 'geocodejson') {
+                unset($oResult['langaddress']);
+                $oResult['name'] = $oPlace['langaddress'];
+            }
+
+            $aOutlineResult = $oPlaceLookup->getOutlines(
+                $oPlace['place_id'],
+                $oPlace['lon'],
+                $oPlace['lat'],
+                Nominatim\ClassTypes\getProperty($oPlace, 'defdiameter', 0.0001)
+            );
+
+            if ($aOutlineResult) {
+                $oResult = array_merge($oResult, $aOutlineResult);
+            }
+
             $aSearchResults[] = $oResult;
         }
     }

website/search.php

@@ -50,6 +50,7 @@ $oGeocode->setQueryFromParams($oParams);
 
 if (!$oGeocode->getQueryString()
     && isset($_SERVER['PATH_INFO'])
+    && strlen($_SERVER['PATH_INFO']) > 0
     && $_SERVER['PATH_INFO'][0] == '/'
 ) {
     $sQuery = substr(rawurldecode($_SERVER['PATH_INFO']), 1);