update change log and authors for pending release

Data for gb_postcode now needs to be downloaded first.
Also fixes table creation for postgis 2.0.

AUTHORS

@@ -12,3 +12,5 @@ Nominatim was written by:
   appelflap
   b3nn0
   Spin0us
+  Kurt Roeckx
+  Rodolphe Quiédeville

ChangeLog

@@ -26,3 +26,19 @@
  * support for US ZIP+4 codes
  * refactoring of front-end PHP code
  * lots of smaller bug fixes
+
+ 2.2
+
+ * correct database rights for www-data
+ * add timestamps for update output
+ * load postgis via extension for postgis >= 2.0
+ * remove non-admin boundaries from addresses
+ * further improve ordering of results with same importance
+ * merge addr:postcode tags into object addresses
+ * include rank and importance in reverse geocode output
+ * replace ST_Line_Interpolate_Point with ST_LineInterpolatePoint
+  (for postgis >= 2.1)
+ * update osm2pgsql to latest version
+ * properly detect changes of admin_level
+ * remove landuses when name is removed
+ * smaller fixes

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT(Nominatim,2.1)
+AC_INIT(Nominatim,2.2)
 if git rev-parse HEAD 2>/dev/null >/dev/null; then
     AC_SUBST([PACKAGE_VERSION], [$PACKAGE_VERSION-git-`git rev-parse --short HEAD`])
 fi

data/gb_postcode_table.sql

@@ -0,0 +1,27 @@
+-- This data contains Ordnance Survey data © Crown copyright and database right 2010. 
+-- Code-Point Open contains Royal Mail data © Royal Mail copyright and database right 2010.
+-- OS data may be used under the terms of the OS OpenData licence:
+-- http://www.ordnancesurvey.co.uk/oswebsite/opendata/licence/docs/licence.pdf
+
+SET statement_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = off;
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+SET escape_string_warning = off;
+
+SET search_path = public, pg_catalog;
+
+SET default_tablespace = '';
+
+SET default_with_oids = false;
+
+CREATE TABLE gb_postcode (
+    id integer,
+    postcode character varying(9),
+    geometry geometry,
+    CONSTRAINT enforce_dims_geometry CHECK ((st_ndims(geometry) = 2)),
+    CONSTRAINT enforce_srid_geometry CHECK ((st_srid(geometry) = 4326))
+);
+
+GRANT SELECT ON TABLE gb_postcode TO "www-data";

lib/Geocode.php

@@ -20,6 +20,7 @@
 		protected $iLimit = 20;
 		protected $iFinalLimit = 10;
 		protected $iOffset = 0;
+		protected $bFallback = false;
 
 		protected $aCountryCodes = false;
 		protected $aNearPoint = false;
@@ -32,6 +33,7 @@
 		protected $iMinAddressRank = 0;
 		protected $iMaxAddressRank = 30;
 		protected $aAddressRankList = array();
+		protected $exactMatchCache = array();
 
 		protected $sAllowedTypesSQLList = false;
 
@@ -117,6 +119,11 @@
 			$this->iOffset = $iOffset;
 		}
 
+		function setFallback($bFallback = true)
+		{
+			$this->bFallback = (bool)$bFallback;
+		}
+
 		function setExcludedPlaceIDs($a)
 		{
 			// TODO: force to int
@@ -213,6 +220,11 @@
 		{
 			$this->sQuery = false;
 
+			// Reset
+			$this->iMinAddressRank = 0;
+			$this->iMaxAddressRank = 30;
+			$this->aAddressRankList = array();
+
 			$this->aStructuredQuery = array();
 			$this->sAllowedTypesSQLList = '';
 
@@ -232,7 +244,29 @@
 					$sAllowedTypesSQLList = '(\'place\',\'boundary\')';
 				}
 			}
+		}
 
+		function fallbackStructuredQuery()
+		{
+			if (!$this->aStructuredQuery) return false;
+
+			$aParams = $this->aStructuredQuery;
+
+			if (sizeof($aParams) == 1) return false;
+
+			$aOrderToFallback = array('postalcode', 'street', 'city', 'county', 'state');
+
+			foreach($aOrderToFallback as $sType)
+			{
+				if (isset($aParams[$sType]))
+				{
+					unset($aParams[$sType]);
+					$this->setStructuredQuery(@$aParams['amenity'], @$aParams['street'], @$aParams['city'], @$aParams['county'], @$aParams['state'], @$aParams['country'], @$aParams['postalcode']);
+					return true;
+				}
+			}
+
+			return false;
 		}
 
 		function getDetails($aPlaceIDs)
@@ -244,13 +278,13 @@
 			// Get the details for display (is this a redundant extra step?)
 			$sPlaceIDs = join(',',$aPlaceIDs);
 
-			$sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id,calculated_country_code as country_code,";
+			$sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id, calculated_country_code as country_code,";
 			$sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
 			$sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,";
 			$sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,";
 			$sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 			$sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance, ";
-			$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(placex.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
+			$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(CASE WHEN placex.rank_search < 28 THEN placex.place_id ELSE placex.parent_place_id END) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
 			$sSQL .= "(extratags->'place') as extra_place ";
 			$sSQL .= "from placex where place_id in ($sPlaceIDs) ";
 			$sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank ";
@@ -269,26 +303,26 @@
 			if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank)
 			{
 				$sSQL .= " union ";
-				$sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,";
+				$sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,";
 				$sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
 				$sSQL .= "null as placename,";
 				$sSQL .= "null as ref,";
 				$sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 				$sSQL .= "-0.15 as importance, ";
-				$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
+				$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
 				$sSQL .= "null as extra_place ";
 				$sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) ";
 				$sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
 				$sSQL .= "group by place_id";
 				if (!$this->bDeDupe) $sSQL .= ",place_id";
 				$sSQL .= " union ";
-				$sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,";
+				$sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,";
 				$sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
 				$sSQL .= "null as placename,";
 				$sSQL .= "null as ref,";
 				$sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 				$sSQL .= "-0.10 as importance, ";
-				$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
+				$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
 				$sSQL .= "null as extra_place ";
 				$sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
 				$sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
@@ -309,6 +343,36 @@
 			return $aSearchResults;
 		}
 
+		/* Perform the actual query lookup.
+
+			Returns an ordered list of results, each with the following fields:
+			  osm_type: type of corresponding OSM object
+							N - node
+							W - way
+							R - relation
+							P - postcode (internally computed)
+			  osm_id: id of corresponding OSM object
+			  class: general object class (corresponds to tag key of primary OSM tag)
+			  type: subclass of object (corresponds to tag value of primary OSM tag)
+			  admin_level: see http://wiki.openstreetmap.org/wiki/Admin_level
+			  rank_search: rank in search hierarchy
+							(see also http://wiki.openstreetmap.org/wiki/Nominatim/Development_overview#Country_to_street_level)
+			  rank_address: rank in address hierarchy (determines orer in address)
+			  place_id: internal key (may differ between different instances)
+			  country_code: ISO country code
+			  langaddress: localized full address
+			  placename: localized name of object
+			  ref: content of ref tag (if available)
+			  lon: longitude
+			  lat: latitude
+			  importance: importance of place based on Wikipedia link count
+			  addressimportance: cumulated importance of address elements
+			  extra_place: type of place (for admin boundaries, if there is a place tag)
+			  aBoundingBox: bounding Box
+			  label: short description of the object class/type (English only) 
+			  name: full name (currently the same as langaddress)
+			  foundorder: secondary ordering for places with same importance
+		*/
 		function lookup()
 		{
 			if (!$this->sQuery && !$this->aStructuredQuery) return false;
@@ -353,11 +417,12 @@
 			if ($this->aRoutePoints)
 			{
 				$sViewboxCentreSQL = "ST_SetSRID('LINESTRING(";
-				$bFirst = false;
-				foreach($this->aRouteaPoints as $aPoint)
+				$bFirst = true;
+				foreach($this->aRoutePoints as $aPoint)
 				{
 					if (!$bFirst) $sViewboxCentreSQL .= ",";
 					$sViewboxCentreSQL .= $aPoint[1].' '.$aPoint[0];
+					$bFirst = false;
 				}
 				$sViewboxCentreSQL .= ")'::geometry,4326)";
 
@@ -1115,12 +1180,16 @@
 							$aOrder[] = "$sImportanceSQL DESC";
 							if (sizeof($aSearch['aFullNameAddress']))
 							{
-								$aOrder[] = '(select count(*) from (select unnest(ARRAY['.join($aSearch['aFullNameAddress'],",").']) INTERSECT select unnest(nameaddress_vector))s) DESC';
+								$sExactMatchSQL = '(select count(*) from (select unnest(ARRAY['.join($aSearch['aFullNameAddress'],",").']) INTERSECT select unnest(nameaddress_vector))s) as exactmatch';
+								$aOrder[] = 'exactmatch DESC';
+							} else {
+								$sExactMatchSQL = '0::int as exactmatch';
 							}
 
 							if (sizeof($aTerms))
 							{
-								$sSQL = "select place_id";
+								$sSQL = "select place_id, ";
+								$sSQL .= $sExactMatchSQL;
 								$sSQL .= " from search_name";
 								$sSQL .= " where ".join(' and ',$aTerms);
 								$sSQL .= " order by ".join(', ',$aOrder);
@@ -1148,6 +1217,7 @@
 									//if ($aViewBoxRow['in_small'] == 't') $bViewBoxMatch = 1;
 									//else if ($aViewBoxRow['in_large'] == 't') $bViewBoxMatch = 2;
 									$aPlaceIDs[] = $aViewBoxRow['place_id'];
+									$this->exactMatchCache[$aViewBoxRow['place_id']] = $aViewBoxRow['exactmatch'];
 								}
 							}
 							//var_Dump($aPlaceIDs);
@@ -1374,6 +1444,14 @@
 			// No results? Done
 			if (!sizeof($aSearchResults))
 			{
+				if ($this->bFallback)
+				{
+					if ($this->fallbackStructuredQuery())
+					{
+						return $this->lookup();
+					}
+				}
+
 				return array();
 			}
 
@@ -1545,7 +1623,24 @@
 				$aResult['importance'] = $aResult['importance'] + ($iCountWords*0.1); // 0.1 is a completely arbitrary number but something in the range 0.1 to 0.5 would seem right
 
 				$aResult['name'] = $aResult['langaddress'];
-				$aResult['foundorder'] = -$aResult['addressimportance'];
+				// secondary ordering (for results with same importance (the smaller the better):
+				//   - approximate importance of address parts
+				$aResult['foundorder'] = -$aResult['addressimportance']/10;
+				//   - number of exact matches from the query
+				if (isset($this->exactMatchCache[$aResult['place_id']]))
+					$aResult['foundorder'] -= $this->exactMatchCache[$aResult['place_id']];
+				else if (isset($this->exactMatchCache[$aResult['parent_place_id']]))
+					$aResult['foundorder'] -= $this->exactMatchCache[$aResult['parent_place_id']];
+				//  - importance of the class/type
+				if (isset($aClassType[$aResult['class'].':'.$aResult['type']]['importance'])
+					&& $aClassType[$aResult['class'].':'.$aResult['type']]['importance'])
+				{
+					$aResult['foundorder'] = $aResult['foundorder'] + 0.000001 * $aClassType[$aResult['class'].':'.$aResult['type']]['importance'];
+				}
+				else
+				{
+					$aResult['foundorder'] = $aResult['foundorder'] + 0.001;
+				}
 				$aSearchResults[$iResNum] = $aResult;
 			}
 			uasort($aSearchResults, 'byImportance');

lib/PlaceLookup.php

@@ -41,8 +41,8 @@
 
 			$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
 
-			$sSQL = "select placex.place_id, partition, osm_type, osm_id, class, type, admin_level, housenumber, street, isin, postcode, country_code, extratags, parent_place_id, linked_place_id, rank_address, rank_search";
-			$sSQL .= " importance, indexed_status, indexed_date, wikipedia, calculated_country_code, ";
+			$sSQL = "select placex.place_id, partition, osm_type, osm_id, class, type, admin_level, housenumber, street, isin, postcode, country_code, extratags, parent_place_id, linked_place_id, rank_address, rank_search, ";
+			$sSQL .= " coalesce(importance,0.75-(rank_search::float/40)) as importance, indexed_status, indexed_date, wikipedia, calculated_country_code, ";
 			$sSQL .= " get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
 			$sSQL .= " get_name_by_language(name, $sLanguagePrefArraySQL) as placename,";
 			$sSQL .= " get_name_by_language(name, ARRAY['ref']) as ref,";

lib/ReverseGeocode.php

@@ -95,8 +95,8 @@
 				$sSQL .= ' and rank_search != 28 and rank_search >= '.$iMaxRank;
 				$sSQL .= ' and (name is not null or housenumber is not null)';
 				$sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\')';
-				$sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') ';
 				$sSQL .= ' and indexed_status = 0 ';
+				$sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') ';
 				$sSQL .= ' OR ST_DWithin('.$sPointSQL.', centroid, '.$fSearchDiam.'))';
 				$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1';
 				if (CONST_Debug) var_dump($sSQL);

lib/init-cmd.php

@@ -1,4 +1,9 @@
 <?php
+	if (file_exists(getenv('NOMINATIM_SETTINGS')))
+	{
+		echo 'Using local config: '.getenv('NOMINATIM_SETTINGS');
+		require_once(getenv('NOMINATIM_SETTINGS'));
+	}
 
 	require_once('init.php');
 	require_once('cmd.php');

lib/init-website.php

@@ -54,9 +54,9 @@
 
 	if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit)
 	{
+		header("HTTP/1.0 429 Too Many Requests");
 		echo "Your IP has been blocked. \n";
-		echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n";
-		echo "Information on the Nominatim usage policy can be found here: http://wiki.openstreetmap.org/wiki/Nominatim#Usage_Policy \n";
+		echo CONST_BlockMessage;
 		exit;
 	}
 

lib/lib.php

@@ -846,12 +846,12 @@
 			$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1';
 			//var_dump($sSQL);
 			$aPlace = $oDB->getRow($sSQL);
-			$iPlaceID = $aPlace['place_id'];
-			if (PEAR::IsError($iPlaceID))
+			if (PEAR::IsError($aPlace))
 			{
-				var_Dump($sSQL, $iPlaceID);
+				var_Dump($sSQL, $aPlace);
 				exit;
 			}
+			$iPlaceID = $aPlace['place_id'];
 		}
 
 		// The point we found might be too small - use the address to find what it is a child of

lib/template/address-jsonv2.php

@@ -20,8 +20,14 @@
                 }
 		if (isset($aPlace['lat'])) $aFilteredPlaces['lat'] = $aPlace['lat'];
 		if (isset($aPlace['lon'])) $aFilteredPlaces['lon'] = $aPlace['lon'];
+
+		$aFilteredPlaces['place_rank'] = $aPlace['rank_search'];
+
                 $aFilteredPlaces['category'] = $aPlace['class'];
                 $aFilteredPlaces['type'] = $aPlace['type'];
+
+		$aFilteredPlaces['importance'] = $aPlace['importance'];
+
                 $aFilteredPlaces['addresstype'] = strtolower($aPlace['addresstype']);
 
 		$aFilteredPlaces['display_name'] = $aPlace['langaddress'];

lib/template/address-xml.php

@@ -22,7 +22,8 @@
 	{
 		echo "<result";
 		if ($aPlace['place_id']) echo ' place_id="'.$aPlace['place_id'].'"';
-		if ($aPlace['osm_type'] && $aPlace['osm_id']) echo ' osm_type="'.($aPlace['osm_type']=='N'?'node':($aPlace['osm_type']=='W'?'way':'relation')).'"'.' osm_id="'.$aPlace['osm_id'].'"';
+		$sOSMType = ($aPlace['osm_type'] == 'N'?'node':($aPlace['osm_type'] == 'W'?'way':($aPlace['osm_type'] == 'R'?'relation':'')));
+		if ($sOSMType) echo ' osm_type="'.$sOSMType.'"'.' osm_id="'.$aPlace['osm_id'].'"';
 		if ($aPlace['ref']) echo ' ref="'.htmlspecialchars($aPlace['ref']).'"';
 		if (isset($aPlace['lat'])) echo ' lat="'.htmlspecialchars($aPlace['lat']).'"';
 		if (isset($aPlace['lon'])) echo ' lon="'.htmlspecialchars($aPlace['lon']).'"';

lib/template/search-xml.php

@@ -80,7 +80,7 @@
 		echo " display_name='".htmlspecialchars($aResult['name'], ENT_QUOTES)."'";
 
 		echo " class='".htmlspecialchars($aResult['class'])."'";
-		echo " type='".htmlspecialchars($aResult['type'])."'";
+		echo " type='".htmlspecialchars($aResult['type'], ENT_QUOTES)."'";
 		echo " importance='".htmlspecialchars($aResult['importance'])."'";
 		if (isset($aResult['icon']) && $aResult['icon'])
 		{

munin/nominatim_importlag

@@ -5,26 +5,42 @@
 # Can be configured through libpq environment variables, for example
 # PGUSER, PGDATABASE, etc. See man page of psql for more information
 #
-# Parameters: 
+# To configure munin for a default installation, add these lines to
+# the file /etc/munin/plugin-conf.d/munin-node or in any file in the
+# directory /etc/munin/plugin-conf.d/
+#
+# [nominatim_*]
+# user www-data
+# env.PGUSER www-data
+# env.PGPORT 5432
+# env.PGDATABASE nominatim
+# env.age_warning 21600
+# env.age_critical 86400
+
+# Parameters:
 #
 #       config   (required)
 #       autoconf (optional - used by munin-config)
 #
- 
+
+. $MUNIN_LIBDIR/plugins/plugin.sh
+
 if [ "$1" = "config" ]; then
- 
-        echo 'graph_title Data import lag' 
-        echo 'graph_args --base 1000 -l 0' 
-        echo 'graph_vlabel minutes' 
-        echo 'graph_category nominatim' 
-        echo 'age.label DB import age' 
-        echo 'age.type GAUGE' 
+
+        echo 'graph_title Data import lag'
+        echo 'graph_args --base 1000 -l 0'
+        echo 'graph_vlabel minutes'
+        echo 'graph_category nominatim'
+        echo 'age.label DB import age'
+        echo 'age.type GAUGE'
         echo 'age.cdef age,60,/'
+        print_warning age
+        print_critical age
         exit 0
 fi
- 
+
 
 delay=`psql -c "copy (select extract(epoch from timezone('utc', now())-lastimportdate)::int from import_status) to stdout"`
- 
- 
+
+
 echo "age.value $delay"

nominatim/nominatim.c

@@ -115,7 +115,7 @@ int main(int argc, char *argv[])
 
     PGconn *conn;
 
-    fprintf(stderr, "nominatim SVN version %s\n\n", VERSION);
+    fprintf(stderr, "nominatim version %s\n\n", VERSION);
 
     while (1)
     {

settings/settings.php

@@ -6,6 +6,7 @@
 	@define('CONST_Debug', false);
 	@define('CONST_Database_DSN', 'pgsql://@/nominatim'); // <driver>://<username>:<password>@<host>:<port>/<database>
 	@define('CONST_Max_Word_Frequency', '50000');
+	@define('CONST_Limit_Reindexing', true);
 
 	// Software versions
 	@define('CONST_Postgresql_Version', '9.1'); // values: 8.3, 8.4, 9.0, 9.1, 9.2
@@ -55,6 +56,7 @@
 	@define('CONST_ClosedForIndexingExceptionIPs', '');
 	@define('CONST_BlockedIPs', '');
 	@define('CONST_BulkUserIPs', '');
+	@define('CONST_BlockMessage', ''); // additional info to show for blocked IPs
 
 	@define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/');
 	@define('CONST_Tile_Default', 'Mapnik');

sql/functions.sql

@@ -347,7 +347,7 @@ BEGIN
 
     w := getorcreate_word_id(s);
 
-    IF NOT (ARRAY[w] <@ result) THEN
+    IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
       result := result || w;
     END IF;
 
@@ -415,7 +415,7 @@ BEGIN
 
   w := getorcreate_word_id(s);
 
-  IF NOT (ARRAY[w] <@ result) THEN
+  IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
     result := result || w;
   END IF;
 
@@ -895,7 +895,7 @@ BEGIN
                 insert into placex (osm_type, osm_id, class, type, admin_level, housenumber, street, addr_place, isin, postcode,
                   country_code, parent_place_id, rank_address, rank_search, indexed_status, geometry)
                   values ('N',prevnode.osm_id, 'place', 'house', prevnode.admin_level, housenum, prevnode.street, prevnode.addr_place, prevnode.isin, coalesce(prevnode.postcode, defpostalcode),
-                  prevnode.country_code, prevnode.parent_place_id, prevnode.rank_address, prevnode.rank_search, 1, ST_Line_Interpolate_Point(linegeo, (housenum::float-orginalstartnumber::float)/originalnumberrange::float));
+                  prevnode.country_code, prevnode.parent_place_id, prevnode.rank_address, prevnode.rank_search, 1, ST_LineInterpolatePoint(linegeo, (housenum::float-orginalstartnumber::float)/originalnumberrange::float));
                 newpoints := newpoints + 1;
 --RAISE WARNING 'interpolation number % % ',prevnode.place_id,housenum;
               END LOOP;
@@ -1123,7 +1123,11 @@ BEGIN
         return NULL;
       END IF;
       NEW.rank_search := NEW.admin_level * 2;
-      NEW.rank_address := NEW.rank_search;
+      IF NEW.type = 'administrative' THEN
+        NEW.rank_address := NEW.rank_search;
+      ELSE
+        NEW.rank_address := 0;
+      END IF;
     ELSEIF NEW.class = 'landuse' AND ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') THEN
       NEW.rank_search := 22;
       NEW.rank_address := NEW.rank_search;
@@ -1241,7 +1245,7 @@ BEGIN
    -- Note: won't work on initial import because the classtype tables
    -- do not yet exist. It won't hurt either.
   classtable := 'place_classtype_' || NEW.class || '_' || NEW.type;
-  SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable INTO result;
+  SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable and schemaname = current_schema() INTO result;
   IF result THEN
     EXECUTE 'INSERT INTO ' || classtable::regclass || ' (place_id, centroid) VALUES ($1,$2)' 
     USING NEW.place_id, ST_Centroid(NEW.geometry);
@@ -1480,9 +1484,9 @@ BEGIN
             FOR relation IN SELECT place_id FROM placex
               WHERE ST_DWithin(location.geometry, placex.geometry, 0.001) and placex.rank_search = 26
                 and st_geometrytype(location.geometry) in ('ST_LineString')
-              ORDER BY (ST_distance(placex.geometry, ST_Line_Interpolate_Point(location.geometry,0))+
-                        ST_distance(placex.geometry, ST_Line_Interpolate_Point(location.geometry,0.5))+
-                        ST_distance(placex.geometry, ST_Line_Interpolate_Point(location.geometry,1))) ASC limit 1
+              ORDER BY (ST_distance(placex.geometry, ST_LineInterpolatePoint(location.geometry,0))+
+                        ST_distance(placex.geometry, ST_LineInterpolatePoint(location.geometry,0.5))+
+                        ST_distance(placex.geometry, ST_LineInterpolatePoint(location.geometry,1))) ASC limit 1
             LOOP
 --RAISE WARNING 'using nearest street to address interpolation line,0.001 %',relation;
               NEW.parent_place_id := relation.place_id;
@@ -1949,7 +1953,7 @@ BEGIN
     -- mark everything linked to this place for re-indexing
     --DEBUG: RAISE WARNING 'placex_delete:03 % %',OLD.osm_type,OLD.osm_id;
     UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = OLD.place_id 
-      and placex.place_id = place_addressline.place_id and indexed_status = 0;
+      and placex.place_id = place_addressline.place_id and indexed_status = 0 and place_addressline.isaddress;
 
     --DEBUG: RAISE WARNING 'placex_delete:04 % %',OLD.osm_type,OLD.osm_id;
     DELETE FROM place_addressline where address_place_id = OLD.place_id;
@@ -1983,7 +1987,7 @@ BEGIN
 
   -- remove from tables for special search
   classtable := 'place_classtype_' || OLD.class || '_' || OLD.type;
-  SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable INTO b;
+  SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable and schemaname = current_schema() INTO b;
   IF b THEN
     EXECUTE 'DELETE FROM ' || classtable::regclass || ' WHERE place_id = $1' USING OLD.place_id;
   END IF;
@@ -2041,6 +2045,8 @@ BEGIN
 
   -- Just block these - lots and pointless
   IF NEW.class in ('landuse','natural') and NEW.name is null THEN
+    -- if the name tag was removed, older versions might still be lurking in the place table
+    DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
     RETURN null;
   END IF;
 
@@ -2086,8 +2092,19 @@ BEGIN
   DELETE from import_polygon_error where osm_type = NEW.osm_type and osm_id = NEW.osm_id;
   DELETE from import_polygon_delete where osm_type = NEW.osm_type and osm_id = NEW.osm_id;
 
-  -- To paraphrase, if there isn't an existing item, OR if the admin level has changed, OR if it is a major change in geometry
-  IF existingplacex.osm_type IS NULL THEN
+  -- To paraphrase, if there isn't an existing item, OR if the admin level has changed
+  IF existingplacex.osm_type IS NULL OR
+    (coalesce(existingplacex.admin_level, 15) != coalesce(NEW.admin_level, 15) AND existingplacex.class = 'boundary' AND existingplacex.type = 'administrative')
+  THEN
+
+    IF existingplacex.osm_type IS NOT NULL THEN
+      -- sanity check: ignore admin_level changes on places with too many active children
+      -- or we end up reindexing entire countries because somebody accidentally deleted admin_level
+      --LIMIT INDEXING: SELECT count(*) FROM (SELECT 'a' FROM placex , place_addressline where address_place_id = existingplacex.place_id and placex.place_id = place_addressline.place_id and indexed_status = 0 and place_addressline.isaddress LIMIT 100001) sub INTO i;
+      --LIMIT INDEXING: IF i > 100000 THEN
+      --LIMIT INDEXING:  RETURN null;
+      --LIMIT INDEXING: END IF;
+    END IF;
 
     IF existing.osm_type IS NOT NULL THEN
       -- pathological case caused by the triggerless copy into place during initial import
@@ -2477,7 +2494,7 @@ BEGIN
       CASE WHEN class = 'place' and type = 'postcode' THEN hstore('name', postcode) ELSE name END as name,
       class, type, admin_level, fromarea, isaddress,
       CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address,
-      distance,calculated_country_code
+      distance,calculated_country_code,postcode
       from place_addressline join placex on (address_place_id = placex.place_id) 
       where place_addressline.place_id = for_place_id 
       and (cached_rank_address > 0 AND cached_rank_address < searchrankaddress)
@@ -2492,6 +2509,9 @@ BEGIN
     IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN
       location.isaddress := FALSE;
     END IF;
+    IF searchpostcode IS NULL and location.isaddress and location.type != 'postcode' and location.postcode IS NOT NULL THEN
+      searchpostcode := location.postcode;
+    END IF;
     IF location.rank_address = 4 AND location.isaddress THEN
       hadcountry := true;
     END IF;

sql/partition-functions.src.sql

@@ -327,9 +327,9 @@ BEGIN
     RETURN;
   END IF;
 
-  p1 := ST_Line_Interpolate_Point(line,0);
-  p2 := ST_Line_Interpolate_Point(line,0.5);
-  p3 := ST_Line_Interpolate_Point(line,1);
+  p1 := ST_LineInterpolatePoint(line,0);
+  p2 := ST_LineInterpolatePoint(line,0.5);
+  p3 := ST_LineInterpolatePoint(line,1);
 
 -- start
   IF in_partition = -partition- THEN

sql/tables.sql

@@ -212,16 +212,9 @@ CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name
 DROP SEQUENCE seq_place;
 CREATE SEQUENCE seq_place start 1;
 GRANT SELECT on placex to "www-data" ;
-GRANT UPDATE ON placex to "www-data" ;
 GRANT SELECT ON search_name to "www-data" ;
-GRANT DELETE on search_name to "www-data" ;
-GRANT INSERT on search_name to "www-data" ;
 GRANT SELECT on place_addressline to "www-data" ;
-GRANT INSERT ON place_addressline to "www-data" ;
-GRANT DELETE on place_addressline to "www-data" ;
 GRANT SELECT ON seq_word to "www-data" ;
-GRANT UPDATE ON seq_word to "www-data" ;
-GRANT INSERT ON word to "www-data" ;
 GRANT SELECT ON planet_osm_ways to "www-data" ;
 GRANT SELECT ON planet_osm_rels to "www-data" ;
 GRANT SELECT on location_area to "www-data" ;

sql/tiger_import_start.sql

@@ -79,7 +79,7 @@ BEGIN
   FOR housenum IN startnumber..endnumber BY stepsize LOOP
     insert into location_property_tiger_import (place_id, partition, parent_place_id, housenumber, postcode, centroid)
     values (nextval('seq_place'), out_partition, out_parent_place_id, housenum, in_postcode,
-      ST_Line_Interpolate_Point(linegeo, (housenum::float-rangestartnumber::float)/numberrange::float));
+      ST_LineInterpolatePoint(linegeo, (housenum::float-rangestartnumber::float)/numberrange::float));
     newpoints := newpoints + 1;
   END LOOP;
 

utils/setup.php

@@ -125,7 +125,17 @@
 			pgsqlRunScript('CREATE EXTENSION hstore');
 		}
 
-		pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
+		$fPostgisVersion = (float) CONST_Postgis_Version;
+		if ($fPostgisVersion < 2.0) {
+			pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
+			pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
+		} else {
+			pgsqlRunScript('CREATE EXTENSION postgis');
+		}
+		if ($fPostgisVersion < 2.1) {
+			// Function was renamed in 2.1 and throws an annoying deprecation warning
+			pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
+		}
 		$sVersionString = $oDB->getOne('select postgis_full_version()');
 		preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 		if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
@@ -134,11 +144,18 @@
 			exit;
 		}
 
-		pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
-		pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode.sql');
+		pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
+		if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz'))
+		{
+			pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
+		}
+		else
+		{
+			echo "WARNING: external UK postcode table not found.\n";
+		}
 		pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
@@ -188,6 +205,7 @@
 		$sTemplate = str_replace('{modulepath}', CONST_BasePath.'/module', $sTemplate);
 		if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
 		if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
+		if (CONST_Limit_Reindexing) $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
 		pgsqlRunScript($sTemplate);
 	}
 
@@ -667,10 +685,29 @@
 		// Convert database DSN to psql parameters
 		$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 		if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
-		$sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -f '.$sFilename;
+		$sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'];
+
+		$ahGzipPipes = null;
+		if (preg_match('/\\.gz$/', $sFilename))
+		{
+			$aDescriptors = array(
+				0 => array('pipe', 'r'),
+				1 => array('pipe', 'w'),
+				2 => array('file', '/dev/null', 'a')
+			);
+			$hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes);
+			if (!is_resource($hGzipProcess)) fail('unable to start zcat');
+			$aReadPipe = $ahGzipPipes[1];
+			fclose($ahGzipPipes[0]);
+		}
+		else
+		{
+			$sCMD .= ' -f '.$sFilename;
+			$aReadPipe = array('pipe', 'r');
+		}
 
 		$aDescriptors = array(
-			0 => array('pipe', 'r'),
+			0 => $aReadPipe,
 			1 => array('pipe', 'w'),
 			2 => array('file', '/dev/null', 'a')
 		);
@@ -678,7 +715,6 @@
 		$hProcess = proc_open($sCMD, $aDescriptors, $ahPipes);
 		if (!is_resource($hProcess)) fail('unable to start pgsql');
 
-		fclose($ahPipes[0]);
 
 		// TODO: error checking
 		while(!feof($ahPipes[1]))
@@ -688,6 +724,12 @@
 		fclose($ahPipes[1]);
 
 		proc_close($hProcess);
+		if ($ahGzipPipes)
+		{
+			fclose($ahGzipPipes[1]);
+			proc_close($hGzipProcess);
+		}
+
 	}
 
 	function pgsqlRunScript($sScript)

utils/tigerAddressImport.py

@@ -3325,6 +3325,102 @@ county_fips = {
 '56041' : 'Uinta, WY' ,
 '56043' : 'Washakie, WY' ,
 '56045' : 'Weston, WY' ,
+'60000' : 'AMERICAN SAMOA',
+'60010' : 'Eastern district, AS' ,
+'60020' : 'Manua district, AS' ,
+'60030' : 'Rose island, AS' ,
+'60040' : 'Swains island, AS' ,
+'60050' : 'Western district, AS' ,
+'66000' : 'GUAM' ,
+'66010' : 'Guam, GU' ,
+'69000' : 'COMMONWEALTH OF THE NORTHERN MARIANA ISLANDS' ,
+'69085' : 'Commonwealth of the Northern Mariana Islands, MP' ,
+'69100' : 'Commonwealth of the Northern Mariana Islands, MP' ,
+'69110' : 'Commonwealth of the Northern Mariana Islands, MP' ,
+'69120' : 'Commonwealth of the Northern Mariana Islands, MP' ,
+'72000' : 'PUERTO RICO', 
+'72001' : 'Adjuntas, PR', 
+'72003' : 'Aguada, PR', 
+'72005' : 'Aguadilla, PR', 
+'72007' : 'Aguas Buenas, PR', 
+'72009' : 'Aibonito, PR', 
+'72011' : 'Anasco, PR', 
+'72013' : 'Arecibo, PR', 
+'72015' : 'Arroyo, PR', 
+'72017' : 'Barceloneta, PR', 
+'72019' : 'Barranquitas, PR', 
+'72021' : 'Bayamon, PR', 
+'72023' : 'Cabo Rojo, PR', 
+'72025' : 'Caguas, PR', 
+'72027' : 'Camuy, PR', 
+'72029' : 'Canovanas, PR', 
+'72031' : 'Carolina, PR', 
+'72033' : 'Catano, PR', 
+'72035' : 'Cayey, PR', 
+'72037' : 'Ceiba, PR', 
+'72039' : 'Ciales, PR', 
+'72041' : 'Cidra, PR', 
+'72043' : 'Coamo, PR', 
+'72045' : 'Comerio, PR', 
+'72047' : 'Corozal, PR', 
+'72049' : 'Culebra, PR', 
+'72051' : 'Dorado, PR', 
+'72053' : 'Fajardo, PR', 
+'72054' : 'Florida, PR', 
+'72055' : 'Guanica, PR', 
+'72057' : 'Guayama, PR', 
+'72059' : 'Guayanilla, PR', 
+'72061' : 'Guaynabo, PR', 
+'72063' : 'Gurabo, PR', 
+'72065' : 'Hatillo, PR', 
+'72067' : 'Hormigueros, PR', 
+'72069' : 'Humacao, PR', 
+'72071' : 'Isabela, PR', 
+'72073' : 'Jayuya, PR', 
+'72075' : 'Juana Diaz, PR', 
+'72077' : 'Juncos, PR', 
+'72079' : 'Lajas, PR', 
+'72081' : 'Lares, PR', 
+'72083' : 'Las Marias, PR', 
+'72085' : 'Las Piedras, PR', 
+'72087' : 'Loiza, PR', 
+'72089' : 'Luquillo, PR', 
+'72091' : 'Manati, PR', 
+'72093' : 'Maricao, PR', 
+'72095' : 'Maunabo, PR', 
+'72097' : 'Mayaguez, PR', 
+'72099' : 'Moca, PR', 
+'72101' : 'Morovis, PR', 
+'72103' : 'Naguabo, PR', 
+'72105' : 'Naranjito, PR', 
+'72107' : 'Orocovis, PR', 
+'72109' : 'Patillas, PR', 
+'72111' : 'Penuelas, PR', 
+'72113' : 'Ponce, PR', 
+'72115' : 'Quebradillas, PR', 
+'72117' : 'Rincon, PR', 
+'72119' : 'Rio Grande, PR', 
+'72121' : 'Sabana Grande, PR', 
+'72123' : 'Salinas, PR', 
+'72125' : 'San German, PR', 
+'72127' : 'San Juan, PR', 
+'72129' : 'San Lorenzo, PR', 
+'72131' : 'San Sebastian, PR', 
+'72133' : 'Santa Isabel, PR', 
+'72135' : 'Toa Alta, PR', 
+'72137' : 'Toa Baja, PR', 
+'72139' : 'Trujillo Alto, PR', 
+'72141' : 'Utuado, PR', 
+'72143' : 'Vega Alta, PR', 
+'72145' : 'Vega Baja, PR', 
+'72147' : 'Vieques, PR', 
+'72149' : 'Villalba, PR', 
+'72151' : 'Yabucoa, PR', 
+'72153' : 'Yauco, PR', 
+'78000' : 'VIRGIN ISLANDS' ,
+'78010' : 'St. Croix, VI' ,
+'78020' : 'St. John, VI' ,
+'78030' : 'St. Thomas, VI' ,
 }
 
 def fipsstate(fips,countyfp):

utils/update.php

@@ -74,6 +74,7 @@
 	$oDB =& getDB();
 
 	$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
+	if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 
 	// cache memory to be used by osm2pgsql, should not be more than the available memory
 	$iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
@@ -82,7 +83,7 @@
 		$iCacheMemory = getCacheMemoryMB();
 		echo "WARNING: resetting cache memory to $iCacheMemory\n";
 	}
-	$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'];
+	$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
 	if (!is_null(CONST_Osm2pgsql_Flatnode_File))
 	{
 		$sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
@@ -360,7 +361,7 @@
 
 	if ($aResult['index'])
 	{
-		passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
+		passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
 	}
 
 	if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
@@ -377,7 +378,7 @@
 		$sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
 		$sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
 		$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
-		$sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'];
+		$sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
 		if (!$aResult['no-npi']) {
 			$sCMDIndex .= '-F ';
 		}
@@ -417,8 +418,9 @@
 							exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
 						}
 						// There are new replication files - use osmosis to download the file
-						echo "\nReplication Delay is ".$aReplicationLag[0]."\n";
+						echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
 					}
+					$fStartTime = time();
 					$fCMDStartTime = time();
 					echo $sCMDDownload."\n";
 					exec($sCMDDownload, $sJunk, $iErrorLevel);
@@ -431,9 +433,10 @@
 					}
 					$iFileSize = filesize($sImportFile);
 					$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
-					echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 					$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
+					var_Dump($sSQL);
 					$oDB->query($sSQL);
+					echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 				}
 
 				$iFileSize = filesize($sImportFile);
@@ -448,10 +451,10 @@
 					echo "Error: $iErrorLevel\n";
 					exit($iErrorLevel);
 				}
-				echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 				$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
 				var_Dump($sSQL);
 				$oDB->query($sSQL);
+				echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 
 				// Archive for debug?
 				unlink($sImportFile);
@@ -461,10 +464,10 @@
 
 			// Index file
 			$sThisIndexCmd = $sCMDIndex;
+			$fCMDStartTime = time();
 
 			if (!$aResult['no-npi'])
 			{
-				$fCMDStartTime = time();
 				$iFileID = $oDB->getOne('select nextval(\'file\')');
 				if (PEAR::isError($iFileID))
 				{
@@ -518,15 +521,16 @@
 				}
 			}
 
-			echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 			$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
+			var_Dump($sSQL);
 			$oDB->query($sSQL);
+			echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 
 			$sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
 			$oDB->query($sSQL);
 
 			$fDuration = time() - $fStartTime;
-			echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n";
+			echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
 			if (!$aResult['import-osmosis-all']) exit;
 
 			if ( CONST_Replication_Update_Interval > 60 )
@@ -537,7 +541,7 @@
 			{
 				$iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
 			}
-			echo "Sleeping $iSleep seconds\n";
+			echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
 			sleep($iSleep);
 		}
 

utils/warm.php

@@ -0,0 +1,63 @@
+#!/usr/bin/php -Cq
+<?php
+
+        require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
+        ini_set('memory_limit', '800M');
+
+	$aCMDOptions = array(
+		"Tools to warm nominatim db",
+		array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+		array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+		array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+		array('reverse-only', '', 0, 1, 0, 0, 'bool', 'Warm reverse only'),
+		array('search-only', '', 0, 1, 0, 0, 'bool', 'Warm reverse only'),
+	);
+	getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
+
+	require_once(CONST_BasePath.'/lib/log.php');
+	require_once(CONST_BasePath.'/lib/Geocode.php');
+	require_once(CONST_BasePath.'/lib/PlaceLookup.php');
+	require_once(CONST_BasePath.'/lib/ReverseGeocode.php');
+
+	$oDB =& getDB();
+
+	$bVerbose = $aResult['verbose'];
+
+	if (!$aResult['search-only']) {
+
+		$oReverseGeocode = new ReverseGeocode($oDB);
+		$oReverseGeocode->setIncludeAddressDetails(true);
+
+		echo "Warm reverse: ";
+		if ($bVerbose) echo "\n";
+		for($i = 0; $i < 1000; $i++) {
+			$fLat = rand(-9000, 9000) / 100;
+			$fLon = rand(-18000, 18000) / 100;
+			if ($bVerbose) echo "$fLat, $fLon = ";
+			$oReverseGeocode->setLanguagePreference(array('en'));
+			$oReverseGeocode->setLatLon($fLat, $fLon);
+			$oReverseGeocode->setZoom(20);
+			$aDetails = $oReverseGeocode->lookup();
+			if ($bVerbose) echo $aDetails['langaddress']."\n";
+			else echo ".";
+		}
+		echo "\n";
+	}
+
+	if (!$aResult['reverse-only']) {
+
+		$oGeocode =& new Geocode($oDB);
+
+		echo "Warm search: ";
+		if ($bVerbose) echo "\n";
+		$sSQL = 'select word from word where word is not null order by search_name_count desc limit 1000';
+		foreach($oDB->getCol($sSQL) as $sWord) {
+			if ($bVerbose) echo "$sWord = ";
+			$oGeocode->setLanguagePreference(array('en'));
+			$oGeocode->setQuery($sWord);
+			$aSearchResults = $oGeocode->lookup();
+			if ($bVerbose) echo $aSearchResults[0]['langaddress']."\n";
+			else echo ".";
+		}
+	}
+

website/js/tiles.js

@@ -3,30 +3,6 @@
  */
 OpenLayers.Util.OSM = {};
 
-/**
- * Constant: MISSING_TILE_URL
- * {String} URL of image to display for missing tiles
- */
-OpenLayers.Util.OSM.MISSING_TILE_URL = "http://www.openstreetmap.org/openlayers/img/404.png";
-
-/**
- * Property: originalOnImageLoadError
- * {Function} Original onImageLoadError function.
- */
-OpenLayers.Util.OSM.originalOnImageLoadError = OpenLayers.Util.onImageLoadError;
-
-/**
- * Function: onImageLoadError
- */
-OpenLayers.Util.onImageLoadError = function() {
-    if (this.src.match(/^http:\/\/[abc]\.[a-z]+\.openstreetmap\.org\//)) {
-        this.src = OpenLayers.Util.OSM.MISSING_TILE_URL;
-    } else if (this.src.match(/^http:\/\/[def]\.tah\.openstreetmap\.org\//)) {
-        // do nothing - this layer is transparent
-    } else {
-        OpenLayers.Util.OSM.originalOnImageLoadError;
-    }
-};
 
 /**
  * Class: OpenLayers.Layer.OSM.Mapnik
@@ -44,9 +20,9 @@ OpenLayers.Layer.OSM.Mapnik = OpenLayers.Class(OpenLayers.Layer.OSM, {
      */
     initialize: function(name, options) {
         var url = [
-            "http://a.tile.openstreetmap.org/${z}/${x}/${y}.png",
-            "http://b.tile.openstreetmap.org/${z}/${x}/${y}.png",
-            "http://c.tile.openstreetmap.org/${z}/${x}/${y}.png"
+            "//a.tile.openstreetmap.org/${z}/${x}/${y}.png",
+            "//b.tile.openstreetmap.org/${z}/${x}/${y}.png",
+            "//c.tile.openstreetmap.org/${z}/${x}/${y}.png"
         ];
         options = OpenLayers.Util.extend({ numZoomLevels: 19, buffer: 0,
            attribution : '© <a target="_parent" href="http://www.openstreetmap.org">OpenStreetMap</a> and contributors, under an <a target="_parent" href="http://www.openstreetmap.org/copyright">open license</a>' }, options);

website/search.php

@@ -34,6 +34,8 @@
 		if (isset($aParams['limit'])) $oGeocode->setLimit((int)$aParams['limit']);
 		if (isset($aParams['offset'])) $oGeocode->setOffset((int)$aParams['offset']);
 
+		if (isset($aParams['fallback'])) $oGeocode->setFallback((int)$aParams['fallback']);
+
 		// List of excluded Place IDs - used for more acurate pageing
 		if (isset($aParams['exclude_place_ids']) && $aParams['exclude_place_ids'])
 		{

website/status.php

@@ -3,24 +3,39 @@
 
 	require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
 
+	function statusError($sMsg)
+	{
+		header("HTTP/1.0 500 Internal Server Error");
+		echo "ERROR: ".$sMsg;
+		exit;
+	}
+
 	$oDB =& getDB();
 	if (!$oDB || PEAR::isError($oDB))
 	{
-		echo "ERROR: No database";
-		exit;
+		statusError("No database");
+	}
+
+	$sStandardWord = $oDB->getOne("select make_standard_name('a')");
+	if (PEAR::isError($sStandardWord))
+	{
+		statusError("Module failed");
+	}
+	if ($sStandardWord != 'a')
+	{
+		statusError("Module call failed");
 	}
 
 	$iWordID = $oDB->getOne("select word_id,word_token, word, class, type, country_code, operator, search_name_count from word where word_token in (' a')");
 	if (PEAR::isError($iWordID))
 	{
-		echo "ERROR: Query failed";
-		exit;
+		statusError("Query failed");
 	}
 	if (!$iWordID)
 	{
-		echo "ERROR: No value";
-		exit;
+		statusError("No value");
 	}
+
 	echo "OK";
 	exit;