make import-osmosis(-all) fail with error code!=0

This might happen for nameless landuse/natural objects that are added to place
during initial import but then dropped when being copied to placex.
If they later receive a name, thus becoming valid, then place_insert should
delete the orphan object in place and reinsert it. If they are large enough,
the place_delete trigger prevents them from being removed. The additional
update fools the delete trigger.

AUTHORS

@@ -1 +1,14 @@
-Nominatim was written by Brian Quinion.
+Nominatim was written by:
+
+  Brian Quinion
+  Sarah Hoffmann
+  Frederik Ramm
+  Michael Spreng
+  Daniele Forsi
+  mfn
+  Grant Slater
+  Andree Klattenhoff
+  IrlJidel
+  appelflap
+  b3nn0
+  Spin0us

ChangeLog

@@ -0,0 +1,28 @@
+2.0.1
+
+ * delete outdated entries from location_area_country
+ * remove remaining uses of INTEGER, to allow node ids larger than 2^31
+
+2.1
+
+ * update to openlayers 2.12 (now custom built)
+ * update fallback OSM boundaries
+ * add support for postgresql 9.2/9.3 and postgis 2.x
+ * add structured queries
+ * add options for polygon output in various formats
+   (geojson, svg, kml, postgis text)
+ * maintenance functions for deleting objects and updating regions
+   (plcae_force_update/place_force_delete)
+ * web view for polygons that need deleting
+ * rate limiting using memcache
+ * improve layout of details page
+ * add support for boundary:postal_code
+ * full CORS support
+ * improve parenting of POIs
+ * support for extract daily diffs from Geofabrik
+ * support for addresses without a street
+   (addr:place and conscription number house numbers)
+ * improve layout of word and search_name_* tables
+ * support for US ZIP+4 codes
+ * refactoring of front-end PHP code
+ * lots of smaller bug fixes

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT(Nominatim,1.9)
+AC_INIT(Nominatim,2.1)
 if git rev-parse HEAD 2>/dev/null >/dev/null; then
     AC_SUBST([PACKAGE_VERSION], [$PACKAGE_VERSION-git-`git rev-parse --short HEAD`])
 fi

contrib/openlayers.cfg

@@ -0,0 +1,36 @@
+# This file includes a small subset of OpenLayers code, designed to be
+# integrated into another application. It includes only the Layer types
+# neccesary to create tiled or untiled WMS, and does not include any Controls.
+# This is the result of what was at the time called "Webmap.js" at the FOSS4G
+# Web Mapping BOF.
+
+[first]
+
+[last]
+
+[include]
+OpenLayers/Map.js
+OpenLayers/Kinetic.js
+OpenLayers/Geometry/MultiLineString.js
+OpenLayers/Geometry/MultiPolygon.js
+OpenLayers/Format/WKT.js
+OpenLayers/Layer/OSM.js
+OpenLayers/Layer/Vector.js
+OpenLayers/Layer/SphericalMercator.js
+OpenLayers/Control/Attribution.js
+OpenLayers/Control/KeyboardDefaults.js
+OpenLayers/Control/Navigation.js
+OpenLayers/Control/MousePosition.js
+OpenLayers/Control/PanZoomBar.js
+OpenLayers/Control/Permalink.js
+OpenLayers/Control/TouchNavigation.js
+OpenLayers/Style.js
+OpenLayers/Protocol/HTTP.js
+OpenLayers/Projection.js
+OpenLayers/Renderer/SVG.js
+OpenLayers/Renderer/VML.js
+OpenLayers/Renderer/Canvas.js
+
+[exclude]
+
+

lib/PlaceLookup.php

@@ -0,0 +1,138 @@
+<?php
+	class PlaceLookup
+	{
+		protected $oDB;
+
+		protected $iPlaceID;
+
+		protected $aLangPrefOrder = array();
+
+		protected $bAddressDetails = false;
+
+		function PlaceLookup(&$oDB)
+		{
+			$this->oDB =& $oDB;
+		}
+
+		function setLanguagePreference($aLangPrefOrder)
+		{
+			$this->aLangPrefOrder = $aLangPrefOrder;
+		}
+
+		function setIncludeAddressDetails($bAddressDetails = true)
+		{
+			$this->bAddressDetails = $bAddressDetails;
+		}
+
+		function setPlaceID($iPlaceID)
+		{
+			$this->iPlaceID = $iPlaceID;
+		}
+
+		function setOSMID($sType, $iID)
+		{
+			$sSQL = "select place_id from placex where osm_type = '".pg_escape_string($sType)."' and osm_id = ".(int)$iID." order by type = 'postcode' asc";
+			$this->iPlaceID = $this->oDB->getOne($sSQL);
+		}
+
+		function lookup()
+		{
+			if (!$this->iPlaceID) return null;
+
+			$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
+
+			$sSQL = "select placex.place_id, partition, osm_type, osm_id, class, type, admin_level, housenumber, street, isin, postcode, country_code, extratags, parent_place_id, linked_place_id, rank_address, rank_search";
+			$sSQL .= " importance, indexed_status, indexed_date, wikipedia, calculated_country_code, ";
+			$sSQL .= " get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
+			$sSQL .= " get_name_by_language(name, $sLanguagePrefArraySQL) as placename,";
+			$sSQL .= " get_name_by_language(name, ARRAY['ref']) as ref,";
+			$sSQL .= " st_y(centroid) as lat, st_x(centroid) as lon";
+			$sSQL .= " from placex where place_id = ".(int)$this->iPlaceID;
+			$aPlace = $this->oDB->getRow($sSQL);
+
+			if (!$aPlace['place_id']) return null;
+
+			if ($this->bAddressDetails)
+			{
+				$aAddress = $this->getAddressNames();
+				$aPlace['aAddress'] = $aAddress;
+			}
+
+			$aClassType = getClassTypes();
+			$sAddressType = '';
+			$sClassType = $aPlace['class'].':'.$aPlace['type'].':'.$aPlace['admin_level'];
+			if (isset($aClassType[$sClassType]) && isset($aClassType[$sClassType]['simplelabel']))
+			{
+				$sAddressType = $aClassType[$aClassType]['simplelabel'];
+			}
+			else
+			{
+				$sClassType = $aPlace['class'].':'.$aPlace['type'];
+				if (isset($aClassType[$sClassType]) && isset($aClassType[$sClassType]['simplelabel']))
+					$sAddressType = $aClassType[$sClassType]['simplelabel'];
+				else $sAddressType = $aPlace['class'];
+			}
+
+			$aPlace['addresstype'] = $sAddressType;
+
+			return $aPlace;
+		}
+
+		function getAddressDetails($bAll = false)
+		{
+			if (!$this->iPlaceID) return null;
+
+			$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
+
+			$sSQL = "select *,get_name_by_language(name,$sLanguagePrefArraySQL) as localname from get_addressdata(".$this->iPlaceID.")";
+			if (!$bAll) $sSQL .= " WHERE isaddress OR type = 'country_code'";
+			$sSQL .= " order by rank_address desc,isaddress desc";
+
+			$aAddressLines = $this->oDB->getAll($sSQL);
+			if (PEAR::IsError($aAddressLines))
+			{
+				var_dump($aAddressLines);
+				exit;
+			}
+			return $aAddressLines;
+		}
+
+		function getAddressNames()
+		{
+			$aAddressLines = $this->getAddressDetails(false);;
+
+			$aAddress = array();
+			$aFallback = array();
+			$aClassType = getClassTypes();
+			foreach($aAddressLines as $aLine)
+			{
+				$bFallback = false;
+				$aTypeLabel = false;
+				if (isset($aClassType[$aLine['class'].':'.$aLine['type'].':'.$aLine['admin_level']])) $aTypeLabel = $aClassType[$aLine['class'].':'.$aLine['type'].':'.$aLine['admin_level']];
+				elseif (isset($aClassType[$aLine['class'].':'.$aLine['type']])) $aTypeLabel = $aClassType[$aLine['class'].':'.$aLine['type']];
+				elseif (isset($aClassType['boundary:administrative:'.((int)($aLine['rank_address']/2))]))
+				{
+					$aTypeLabel = $aClassType['boundary:administrative:'.((int)($aLine['rank_address']/2))];
+					$bFallback = true;
+				}
+				else
+				{
+					$aTypeLabel = array('simplelabel'=>'address'.$aLine['rank_address']);
+					$bFallback = true;
+				}
+				if ($aTypeLabel && ((isset($aLine['localname']) && $aLine['localname']) || (isset($aLine['housenumber']) && $aLine['housenumber'])))
+				{
+					$sTypeLabel = strtolower(isset($aTypeLabel['simplelabel'])?$aTypeLabel['simplelabel']:$aTypeLabel['label']);
+					$sTypeLabel = str_replace(' ','_',$sTypeLabel);
+					if (!isset($aAddress[$sTypeLabel]) || (isset($aFallback[$sTypeLabel]) && $aFallback[$sTypeLabel]))
+					{
+						$aAddress[$sTypeLabel] = $aLine['localname']?$aLine['localname']:$aLine['housenumber'];
+					}
+					$aFallback[$sTypeLabel] = $bFallback;
+				}
+			}
+			return $aAddress;
+		}
+
+	}
+?>

lib/ReverseGeocode.php

@@ -0,0 +1,139 @@
+<?php
+	class ReverseGeocode
+	{
+		protected $oDB;
+
+		protected $fLat;
+		protected $fLon;
+		protected $iMaxRank = 28;
+
+		protected $aLangPrefOrder = array();
+
+		protected $bShowAddressDetails = true;
+
+		function ReverseGeocode(&$oDB)
+		{
+			$this->oDB =& $oDB;
+		}
+
+		function setLanguagePreference($aLangPref)
+		{
+			$this->aLangPrefOrder = $aLangPref;
+		}
+
+		function setIncludeAddressDetails($bAddressDetails = true)
+		{
+			$this->bAddressDetails = $bAddressDetails;
+		}
+
+		function setLatLon($fLat, $fLon)
+		{
+			$this->fLat = (float)$fLat;
+			$this->fLon = (float)$fLon;
+		}
+
+		function setRank($iRank)
+		{
+			$this->iMaxRank = $iRank;
+		}
+
+		function setZoom($iZoom)
+		{
+			// Zoom to rank, this could probably be calculated but a lookup gives fine control
+			$aZoomRank = array(
+				0 => 2, // Continent / Sea
+				1 => 2,
+				2 => 2,
+				3 => 4, // Country
+				4 => 4,
+				5 => 8, // State
+				6 => 10, // Region
+				7 => 10,
+				8 => 12, // County
+				9 => 12,
+				10 => 17, // City
+				11 => 17,
+				12 => 18, // Town / Village
+				13 => 18,
+				14 => 22, // Suburb
+				15 => 22,
+				16 => 26, // Street, TODO: major street?
+				17 => 26,
+				18 => 30, // or >, Building
+				19 => 30, // or >, Building
+				);
+			$this->iMaxRank = (isset($iZoom) && isset($aZoomRank[$iZoom]))?$aZoomRank[$iZoom]:28;
+		}
+
+		function lookup()
+		{
+			$sPointSQL = 'ST_SetSRID(ST_Point('.$this->fLon.','.$this->fLat.'),4326)';
+			$iMaxRank = $this->iMaxRank;
+
+			// Find the nearest point
+			$fSearchDiam = 0.0004;
+			$iPlaceID = null;
+			$aArea = false;
+			$fMaxAreaDistance = 1;
+			while(!$iPlaceID && $fSearchDiam < $fMaxAreaDistance)
+			{
+				$fSearchDiam = $fSearchDiam * 2;
+
+				// If we have to expand the search area by a large amount then we need a larger feature
+				// then there is a limit to how small the feature should be
+				if ($fSearchDiam > 2 && $iMaxRank > 4) $iMaxRank = 4;
+				if ($fSearchDiam > 1 && $iMaxRank > 9) $iMaxRank = 8;
+				if ($fSearchDiam > 0.8 && $iMaxRank > 10) $iMaxRank = 10;
+				if ($fSearchDiam > 0.6 && $iMaxRank > 12) $iMaxRank = 12;
+				if ($fSearchDiam > 0.2 && $iMaxRank > 17) $iMaxRank = 17;
+				if ($fSearchDiam > 0.1 && $iMaxRank > 18) $iMaxRank = 18;
+				if ($fSearchDiam > 0.008 && $iMaxRank > 22) $iMaxRank = 22;
+				if ($fSearchDiam > 0.001 && $iMaxRank > 26) $iMaxRank = 26;
+
+				$sSQL = 'select place_id,parent_place_id,rank_search from placex';
+				$sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')';
+				$sSQL .= ' and rank_search != 28 and rank_search >= '.$iMaxRank;
+				$sSQL .= ' and (name is not null or housenumber is not null)';
+				$sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\')';
+				$sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') ';
+				$sSQL .= ' and indexed_status = 0 ';
+				$sSQL .= ' OR ST_DWithin('.$sPointSQL.', centroid, '.$fSearchDiam.'))';
+				$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1';
+				if (CONST_Debug) var_dump($sSQL);
+				$aPlace = $this->oDB->getRow($sSQL);
+				if (PEAR::IsError($aPlace))
+				{
+					failInternalError("Could not determine closest place.", $sSQL, $aPlace);
+				}
+				$iPlaceID = $aPlace['place_id'];
+				$iParentPlaceID = $aPlace['parent_place_id'];
+			}
+
+			// The point we found might be too small - use the address to find what it is a child of
+			if ($iPlaceID && $iMaxRank < 28)
+			{
+				if ($aPlace['rank_search'] > 28 && $iParentPlaceID)
+				{
+					$iPlaceID = $iParentPlaceID;
+				}
+				$sSQL = "select address_place_id from place_addressline where place_id = $iPlaceID order by abs(cached_rank_address - $iMaxRank) asc,cached_rank_address desc,isaddress desc,distance desc limit 1";
+				$iPlaceID = $this->oDB->getOne($sSQL);
+				if (PEAR::IsError($iPlaceID))
+				{
+					failInternalError("Could not get parent for place.", $sSQL, $iPlaceID);
+				}
+				if (!$iPlaceID)
+				{
+					$iPlaceID = $aPlace['place_id'];
+				}
+			}
+
+			$oPlaceLookup = new PlaceLookup($this->oDB);
+			$oPlaceLookup->setLanguagePreference($this->aLangPrefOrder);
+			$oPlaceLookup->setIncludeAddressDetails($this->bAddressDetails);
+			$oPlaceLookup->setPlaceId($iPlaceID);
+
+			return $oPlaceLookup->lookup();
+		}
+	}
+?>

lib/init-website.php

@@ -1,14 +1,58 @@
 <?php
-
 	require_once('init.php');
 
+	if (CONST_NoAccessControl)
+	{
+		header("Access-Control-Allow-Origin: *");
+		header("Access-Control-Allow-Methods: OPTIONS,GET");
+		if (!empty($_SERVER['HTTP_ACCESS_CONTROL_REQUEST_HEADERS']))
+		{
+			header("Access-Control-Allow-Headers: ".$_SERVER['HTTP_ACCESS_CONTROL_REQUEST_HEADERS']);
+		}
+	}
+	if ($_SERVER['REQUEST_METHOD'] == 'OPTIONS') exit;
+
 	if (CONST_ClosedForIndexing && strpos(CONST_ClosedForIndexingExceptionIPs, ','.$_SERVER["REMOTE_ADDR"].',') === false)
  	{
 		echo "Closed for re-indexing...";
 		exit;
 	}
 
-	if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
+	$aBucketKeys = array();
+
+	if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST)));
+	if (isset($_SERVER["REMOTE_ADDR"])) $aBucketKeys[] = $_SERVER["REMOTE_ADDR"];
+	if (isset($_GET["email"])) $aBucketKeys[] = $_GET["email"];
+
+	$fBucketVal = doBucket($aBucketKeys, 
+			(defined('CONST_ConnectionBucket_PageType')?constant('CONST_ConnectionBucket_Cost_'.CONST_ConnectionBucket_PageType):1) + user_busy_cost(),
+			CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
+
+	if ($fBucketVal > CONST_ConnectionBucket_WaitLimit && $fBucketVal < CONST_ConnectionBucket_BlockLimit)
+	{
+		$m = getBucketMemcache();
+		$iCurrentSleeping = $m->increment('sleepCounter');
+		if (false === $iCurrentSleeping)
+		{
+			$m->add('sleepCounter', 0);
+			$iCurrentSleeping = $m->increment('sleepCounter');
+		}
+		if ($iCurrentSleeping >= CONST_ConnectionBucket_MaxSleeping || isBucketSleeping($aBucketKeys))
+		{
+			// Too many threads sleeping already.  This becomes a hard block.
+			$fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_BlockLimit, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
+		}
+		else
+		{
+			setBucketSleeping($aBucketKeys, true);
+			sleep(($fBucketVal - CONST_ConnectionBucket_WaitLimit)/CONST_ConnectionBucket_LeakRate);
+			$fBucketVal = doBucket($aBucketKeys, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_LeakRate, CONST_ConnectionBucket_BlockLimit);
+			setBucketSleeping($aBucketKeys, false);
+		}
+		$m->decrement('sleepCounter');
+	}
+
+	if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit)
 	{
 		echo "Your IP has been blocked. \n";
 		echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n";
@@ -17,3 +61,4 @@
 	}
 
 	header('Content-type: text/html; charset=utf-8');
+

lib/init.php

@@ -4,6 +4,7 @@
 
 	require_once(CONST_BasePath.'/settings/settings.php');
 	require_once(CONST_BasePath.'/lib/lib.php');
+	require_once(CONST_BasePath.'/lib/leakybucket.php');
 	require_once(CONST_BasePath.'/lib/db.php');
 
 	if (get_magic_quotes_gpc())

lib/leakybucket.php

@@ -0,0 +1,168 @@
+<?php
+
+	function getBucketMemcache()
+	{
+		static $m;
+
+		if (!CONST_ConnectionBucket_MemcacheServerAddress) return null;
+		if (!isset($m))
+		{
+		        $m = new Memcached();
+        		$m->addServer(CONST_ConnectionBucket_MemcacheServerAddress, CONST_ConnectionBucket_MemcacheServerPort);
+		}
+		return $m;
+	}
+
+	function doBucket($asKey, $iRequestCost, $iLeakPerSecond, $iThreshold)
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return 0;
+
+		$iMaxVal = 0;
+	        $t = time();
+
+		foreach($asKey as $sKey)
+		{
+		        $aCurrentBlock = $m->get($sKey);
+	        	if (!$aCurrentBlock)
+			{
+		                $aCurrentBlock = array($iRequestCost, $t, false);
+        		}
+			else
+			{
+				// add RequestCost
+				// remove leak * the time since the last request 
+	        		$aCurrentBlock[0] += $iRequestCost - ($t - $aCurrentBlock[1])*$iLeakPerSecond;
+		        	$aCurrentBlock[1] = $t;
+			}
+
+	        	if ($aCurrentBlock[0] <= 0)
+			{
+                		$m->delete($sKey);
+		        }
+			else
+			{
+				// If we have hit the threshold stop and record this to the block list
+				if ($aCurrentBlock[0] >= $iThreshold)
+				{
+					$aCurrentBlock[0] = $iThreshold;
+
+					// Make up to 10 attempts to record this to memcache (with locking to prevent conflicts)
+					$i = 10;
+					for($i = 0; $i < 10; $i++)
+					{
+						$aBlockedList = $m->get('blockedList', null, $hCasToken);
+						if (!$aBlockedList)
+						{
+							$aBlockedList = array();
+							$m->add('blockedList', $aBlockedList);
+							$aBlockedList = $m->get('blockedList', null, $hCasToken);
+						}
+						if (!isset($aBlockedList[$sKey]))
+						{
+							$aBlockedList[$sKey] = array(1, $t);
+						}
+						else
+						{
+							$aBlockedList[$sKey][0]++;
+							$aBlockedList[$sKey][1] = $t;
+						}
+						if (sizeof($aBlockedList) > CONST_ConnectionBucket_MaxBlockList)
+						{
+							uasort($aBlockedList, 'byValue1');
+							$aBlockedList = array_slice($aBlockedList, 0, CONST_ConnectionBucket_MaxBlockList);
+						}
+						$x = $m->cas($hCasToken, 'blockedList', $aBlockedList);
+						if ($x) break;
+					}
+				}
+				// Only keep in memcache until the time it would have expired (to avoid clutering memcache)
+	                	$m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/$iLeakPerSecond);
+			}
+
+			// Bucket result in the largest bucket we find
+			$iMaxVal = max($iMaxVal, $aCurrentBlock[0]);
+		}
+
+		return $iMaxVal;
+        }
+
+	function isBucketSleeping($asKey)
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return false;
+
+		foreach($asKey as $sKey)
+		{
+		        $aCurrentBlock = $m->get($sKey);
+			if ($aCurrentBlock[2]) return true;
+		}
+		return false;
+	}
+
+	function setBucketSleeping($asKey, $bVal)
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return false;
+
+		$iMaxVal = 0;
+	        $t = time();
+
+		foreach($asKey as $sKey)
+		{
+		        $aCurrentBlock = $m->get($sKey);
+			$aCurrentBlock[2] = $bVal;
+			$m->set($sKey, $aCurrentBlock, $t + 1 + $aCurrentBlock[0]/CONST_ConnectionBucket_LeakRate);
+		}
+		return true;
+	}
+
+	function byValue1($a, $b)
+	{
+		if ($a[1] == $b[1])
+		{
+			return 0;
+		}
+		return ($a[1] > $b[1]) ? -1 : 1;
+	}
+
+	function byLastBlockTime($a, $b)
+	{
+		if ($a['lastBlockTimestamp'] == $b['lastBlockTimestamp'])
+		{
+			return 0;
+		}
+		return ($a['lastBlockTimestamp'] > $b['lastBlockTimestamp']) ? -1 : 1;
+	}
+
+	function getBucketBlocks()
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return null;
+	        $t = time();
+		$aBlockedList = $m->get('blockedList', null, $hCasToken);
+		if (!$aBlockedList) $aBlockedList = array();
+		foreach($aBlockedList as $sKey => $aDetails)
+		{
+			$aCurrentBlock = $m->get($sKey);
+			if (!$aCurrentBlock) $aCurrentBlock = array(0, $t);
+			$iCurrentBucketSize = max(0, $aCurrentBlock[0] - ($t - $aCurrentBlock[1])*CONST_ConnectionBucket_LeakRate);
+			$aBlockedList[$sKey] = array(
+				'totalBlocks' => $aDetails[0],
+				'lastBlockTimestamp' => $aDetails[1],
+				'isSleeping' => (isset($aCurrentBlock[2])?$aCurrentBlock[2]:false),
+				'currentBucketSize' => $iCurrentBucketSize,
+				'currentlyBlocked' => $iCurrentBucketSize + (CONST_ConnectionBucket_Cost_Reverse) >= CONST_ConnectionBucket_BlockLimit,
+				);
+		}
+		uasort($aBlockedList, 'byLastBlockTime');
+		return $aBlockedList;
+	}
+
+	function clearBucketBlocks()
+	{
+	        $m = getBucketMemcache();
+		if (!$m) return false;
+		$m->delete('blockedList');
+		return true;
+	}

lib/lib.php

@@ -1,6 +1,6 @@
 <?php
 
-	function failInternalError($sError, $sSQL = false, $vDumpVar = false) 
+	function failInternalError($sError, $sSQL = false, $vDumpVar = false)
 	{
 		header('HTTP/1.0 500 Internal Server Error');
 		header('Content-type: text/html; charset=utf-8');
@@ -11,10 +11,12 @@
 		if (CONST_Debug)
 		{
 			echo "<hr><h2>Debugging Information</h2><br>";
-			if ($sSQL) {
+			if ($sSQL)
+			{
 				echo "<h3>SQL query</h3><code>".$sSQL."</code>";
 			}
-			if ($vDumpVar) {
+			if ($vDumpVar)
+			{
 				echo "<h3>Result</h3> <code>";
 				var_dump($vDumpVar);
 				echo "</code>";
@@ -22,9 +24,22 @@
 		}
 		echo "\n</body></html>\n";
 		exit;
-
 	}
 
+
+	function userError($sError)
+	{
+		header('HTTP/1.0 400 Bad Request');
+		header('Content-type: text/html; charset=utf-8');
+		echo "<html><body><h1>Bad Request</h1>";
+		echo '<p>Nominatim has encountered an error with your request.</p>';
+		echo "<p><b>Details:</b> ".$sError,"</p>";
+		echo '<p>If you feel this error is incorrect feel free to report the bug in the <a href="http://trac.openstreetmap.org">OSM bug database</a>. Please include the error message above and the URL you used.</p>';
+		echo "\n</body></html>\n";
+		exit;
+	}
+
+
 	function fail($sError, $sUserError = false)
 	{
 		if (!$sUserError) $sUserError = $sError;
@@ -33,23 +48,26 @@
 		exit;
 	}
 
+
 	function getBlockingProcesses()
 	{
-                $sStats = file_get_contents('/proc/stat');
-                if (preg_match('/procs_blocked ([0-9]+)/i', $sStats, $aMatches))
-                {
+		$sStats = file_get_contents('/proc/stat');
+		if (preg_match('/procs_blocked ([0-9]+)/i', $sStats, $aMatches))
+		{
 			return (int)$aMatches[1];
-                }
+		}
 		return 0;
 	}
 
+
 	function getLoadAverage()
 	{
 		$sLoadAverage = file_get_contents('/proc/loadavg');
-                $aLoadAverage = explode(' ',$sLoadAverage);
-		return (int)$aLoadAverage[0];
+		$aLoadAverage = explode(' ',$sLoadAverage);
+		return (float)$aLoadAverage[0];
 	}
 
+
 	function getProcessorCount()
 	{
 		$sCPU = file_get_contents('/proc/cpuinfo');
@@ -57,6 +75,7 @@
 		return sizeof($aMatches[0]);
 	}
 
+
 	function getTotalMemoryMB()
 	{
 		$sCPU = file_get_contents('/proc/meminfo');
@@ -64,6 +83,7 @@
 		return (int)($aMatches[1]/1024);
 	}
 
+
 	function getCacheMemoryMB()
 	{
 		$sCPU = file_get_contents('/proc/meminfo');
@@ -71,29 +91,32 @@
 		return (int)($aMatches[1]/1024);
 	}
 
+
 	function bySearchRank($a, $b)
 	{
 		if ($a['iSearchRank'] == $b['iSearchRank']) return 0;
 		return ($a['iSearchRank'] < $b['iSearchRank']?-1:1);
 	}
 
+
 	function byImportance($a, $b)
 	{
 		if ($a['importance'] != $b['importance'])
 			return ($a['importance'] > $b['importance']?-1:1);
-/*
-		if ($a['aPointPolygon']['numfeatures'] != $b['aPointPolygon']['numfeatures'])
-			return ($a['aPointPolygon']['numfeatures'] > $b['aPointPolygon']['numfeatures']?-1:1);
-		if ($a['aPointPolygon']['area'] != $b['aPointPolygon']['area'])
-			return ($a['aPointPolygon']['area'] > $b['aPointPolygon']['area']?-1:1);
-//		if ($a['levenshtein'] != $b['levenshtein'])
-//			return ($a['levenshtein'] < $b['levenshtein']?-1:1);
+		/*
+		   if ($a['aPointPolygon']['numfeatures'] != $b['aPointPolygon']['numfeatures'])
+		   return ($a['aPointPolygon']['numfeatures'] > $b['aPointPolygon']['numfeatures']?-1:1);
+		   if ($a['aPointPolygon']['area'] != $b['aPointPolygon']['area'])
+		   return ($a['aPointPolygon']['area'] > $b['aPointPolygon']['area']?-1:1);
+		//		if ($a['levenshtein'] != $b['levenshtein'])
+		//			return ($a['levenshtein'] < $b['levenshtein']?-1:1);
 		if ($a['rank_search'] != $b['rank_search'])
-			return ($a['rank_search'] < $b['rank_search']?-1:1);
-*/
+		return ($a['rank_search'] < $b['rank_search']?-1:1);
+		 */
 		return ($a['foundorder'] < $b['foundorder']?-1:1);
 	}
 
+
 	function getPreferredLanguages()
 	{
 		// If we have been provided the value in $_GET it overrides browser value
@@ -103,16 +126,19 @@
 		}
 
 		$aLanguages = array();
-		if (preg_match_all('/(([a-z]{1,8})(-[a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i', $_SERVER['HTTP_ACCEPT_LANGUAGE'], $aLanguagesParse, PREG_SET_ORDER))
+		if (isset($_SERVER["HTTP_ACCEPT_LANGUAGE"]))
 		{
-			foreach($aLanguagesParse as $iLang => $aLanguage)
+			if (preg_match_all('/(([a-z]{1,8})(-[a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i', $_SERVER['HTTP_ACCEPT_LANGUAGE'], $aLanguagesParse, PREG_SET_ORDER))
 			{
-				$aLanguages[$aLanguage[1]] = isset($aLanguage[5])?(float)$aLanguage[5]:1 - ($iLang/100);
-				if (!isset($aLanguages[$aLanguage[2]])) $aLanguages[$aLanguage[2]] = $aLanguages[$aLanguage[1]]/10;
+				foreach($aLanguagesParse as $iLang => $aLanguage)
+				{
+					$aLanguages[$aLanguage[1]] = isset($aLanguage[5])?(float)$aLanguage[5]:1 - ($iLang/100);
+					if (!isset($aLanguages[$aLanguage[2]])) $aLanguages[$aLanguage[2]] = $aLanguages[$aLanguage[1]]/10;
+				}
+				arsort($aLanguages);
 			}
-			arsort($aLanguages);
 		}
-		if (!sizeof($aLanguages)) $aLanguages = array(CONST_Default_Language=>1);
+		if (!sizeof($aLanguages) && CONST_Default_Language) $aLanguages = array(CONST_Default_Language=>1);
 		foreach($aLanguages as $sLangauge => $fLangauagePref)
 		{
 			$aLangPrefOrder['short_name:'.$sLangauge] = 'short_name:'.$sLangauge;
@@ -138,23 +164,27 @@
 		return $aLangPrefOrder;
 	}
 
-	function getWordSets($aWords)
+
+	function getWordSets($aWords, $iDepth)
 	{
 		$aResult = array(array(join(' ',$aWords)));
 		$sFirstToken = '';
-		while(sizeof($aWords) > 1)
-		{
-			$sWord = array_shift($aWords);
-			$sFirstToken .= ($sFirstToken?' ':'').$sWord;
-			$aRest = getWordSets($aWords);
-			foreach($aRest as $aSet)
+		if ($iDepth < 8) {
+			while(sizeof($aWords) > 1)
 			{
-				$aResult[] = array_merge(array($sFirstToken),$aSet);
+				$sWord = array_shift($aWords);
+				$sFirstToken .= ($sFirstToken?' ':'').$sWord;
+				$aRest = getWordSets($aWords, $iDepth+1);
+				foreach($aRest as $aSet)
+				{
+					$aResult[] = array_merge(array($sFirstToken),$aSet);
+				}
 			}
 		}
 		return $aResult;
 	}
 
+
 	function getTokensFromSets($aSets)
 	{
 		$aTokens = array();
@@ -170,37 +200,39 @@
 		return $aTokens;
 	}
 
+
 	/*
-		GB Postcode functions
-	*/
+	   GB Postcode functions
+	 */
 
 	function gbPostcodeAlphaDifference($s1, $s2)
 	{
 		$aValues = array(
-			'A'=>0,
-			'B'=>1,
-			'D'=>2,
-			'E'=>3,
-			'F'=>4,
-			'G'=>5,
-			'H'=>6,
-			'J'=>7,
-			'L'=>8,
-			'N'=>9,
-			'O'=>10,
-			'P'=>11,
-			'Q'=>12,
-			'R'=>13,
-			'S'=>14,
-			'T'=>15,
-			'U'=>16,
-			'W'=>17,
-			'X'=>18,
-			'Y'=>19,
-			'Z'=>20);
+				'A'=>0,
+				'B'=>1,
+				'D'=>2,
+				'E'=>3,
+				'F'=>4,
+				'G'=>5,
+				'H'=>6,
+				'J'=>7,
+				'L'=>8,
+				'N'=>9,
+				'O'=>10,
+				'P'=>11,
+				'Q'=>12,
+				'R'=>13,
+				'S'=>14,
+				'T'=>15,
+				'U'=>16,
+				'W'=>17,
+				'X'=>18,
+				'Y'=>19,
+				'Z'=>20);
 		return abs(($aValues[$s1[0]]*21+$aValues[$s1[1]]) - ($aValues[$s2[0]]*21+$aValues[$s2[1]]));
 	}
-	
+
+
 	function gbPostcodeCalculate($sPostcode, $sPostcodeSector, $sPostcodeEnd, &$oDB)
 	{
 		// Try an exact match on the gb_postcode table
@@ -211,7 +243,7 @@
 			var_dump($sSQL, $aNearPostcodes);
 			exit;
 		}
-		
+
 		if (sizeof($aNearPostcodes))
 		{
 			return array(array('lat' => $aNearPostcodes[0]['lat'], 'lon' => $aNearPostcodes[0]['lon'], 'radius' => 0.005));
@@ -220,6 +252,7 @@
 		return false;
 	}
 
+
 	function usPostcodeCalculate($sPostcode, &$oDB)
 	{
 		$iZipcode = (int)$sPostcode;
@@ -235,7 +268,7 @@
 
 		if (!sizeof($aNearPostcodes))
 		{
-   			$sSQL = 'select zipcode,ST_X(ST_Centroid(geometry)) as lon,ST_Y(ST_Centroid(geometry)) as lat from us_zipcode where zipcode between '.($iZipcode-100).' and '.($iZipcode+100).' order by abs(zipcode - '.$iZipcode.') asc limit 5';
+			$sSQL = 'select zipcode,ST_X(ST_Centroid(geometry)) as lon,ST_Y(ST_Centroid(geometry)) as lat from us_zipcode where zipcode between '.($iZipcode-100).' and '.($iZipcode+100).' order by abs(zipcode - '.$iZipcode.') asc limit 5';
 			$aNearPostcodes = $oDB->getAll($sSQL);
 			if (PEAR::IsError($aNearPostcodes))
 			{
@@ -259,7 +292,7 @@
 				$fFac = 1;
 			else
 				$fFac = 1/($iDiff*$iDiff);
-			
+
 			$fTotalFac += $fFac;
 			$fTotalLat += $aPostcode['lat'] * $fFac;
 			$fTotalLon += $aPostcode['lon'] * $fFac;
@@ -273,30 +306,35 @@
 		return false;
 
 		/*
-			$fTotalFac is a surprisingly good indicator of accuracy
-			$iZoom = 18 + round(log($fTotalFac,32));
-			$iZoom = max(13,min(18,$iZoom));
-		*/
+		   $fTotalFac is a surprisingly good indicator of accuracy
+		   $iZoom = 18 + round(log($fTotalFac,32));
+		   $iZoom = max(13,min(18,$iZoom));
+		 */
 	}
 
+
 	function getClassTypes()
 	{
 		return array(
+ 'boundary:administrative:1' => array('label'=>'Continent','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'boundary:administrative:2' => array('label'=>'Country','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'place:country' => array('label'=>'Country','frequency'=>0,'icon'=>'poi_boundary_administrative','defzoom'=>6, 'defdiameter' => 15,),
+ 'boundary:administrative:3' => array('label'=>'State','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'boundary:administrative:4' => array('label'=>'State','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'place:state' => array('label'=>'State','frequency'=>0,'icon'=>'poi_boundary_administrative','defzoom'=>8, 'defdiameter' => 5.12,),
  'boundary:administrative:5' => array('label'=>'State District','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'boundary:administrative:6' => array('label'=>'County','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
+ 'boundary:administrative:7' => array('label'=>'County','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'place:county' => array('label'=>'County','frequency'=>108,'icon'=>'poi_boundary_administrative','defzoom'=>10, 'defdiameter' => 1.28,),
  'boundary:administrative:8' => array('label'=>'City','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'place:city' => array('label'=>'City','frequency'=>66,'icon'=>'poi_place_city','defzoom'=>12, 'defdiameter' => 0.32,),
  'boundary:administrative:9' => array('label'=>'City District','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'boundary:administrative:10' => array('label'=>'Suburb','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'boundary:administrative:11' => array('label'=>'Neighbourhood','frequency'=>0,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
- 'place:region' => array('label'=>'Region','frequency'=>0,'icon'=>'poi_boundary_administrative','defzoom'=>8, 'defdiameter' => 5.12,),
+ 'place:region' => array('label'=>'Region','frequency'=>0,'icon'=>'poi_boundary_administrative','defzoom'=>8, 'defdiameter' => 0.04,),
  'place:island' => array('label'=>'Island','frequency'=>288,'icon'=>'','defzoom'=>11, 'defdiameter' => 0.64,),
  'boundary:administrative' => array('label'=>'Administrative','frequency'=>413,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
+ 'boundary:postal_code' => array('label'=>'Postcode','frequency'=>413,'icon'=>'poi_boundary_administrative', 'defdiameter' => 0.32,),
  'place:town' => array('label'=>'Town','frequency'=>1497,'icon'=>'poi_place_town','defzoom'=>14, 'defdiameter' => 0.08,),
  'place:village' => array('label'=>'Village','frequency'=>11230,'icon'=>'poi_place_village','defzoom'=>15, 'defdiameter' => 0.04,),
  'place:hamlet' => array('label'=>'Hamlet','frequency'=>7075,'icon'=>'poi_place_village','defzoom'=>15, 'defdiameter' => 0.04,),
@@ -443,11 +481,11 @@
  'place:house_number' => array('label'=>'House Number','frequency'=>2086,'icon'=>'','defzoom'=>18,),
  'place:country_code' => array('label'=>'Country Code','frequency'=>2086,'icon'=>'','defzoom'=>18,),
 
-//
+ //
 
  'leisure:pitch' => array('label'=>'Pitch','frequency'=>762,'icon'=>'',),
  'highway:unsurfaced' => array('label'=>'Unsurfaced','frequency'=>492,'icon'=>'',),
- 'historic:ruins' => array('label'=>'Ruins','frequency'=>483,'icon'=>'shopping_jewelry',),
+ 'historic:ruins' => array('label'=>'Ruins','frequency'=>483,'icon'=>'tourist_ruin',),
  'amenity:college' => array('label'=>'College','frequency'=>473,'icon'=>'education_school',),
  'historic:monument' => array('label'=>'Monument','frequency'=>470,'icon'=>'tourist_monument',),
  'railway:subway' => array('label'=>'Subway','frequency'=>385,'icon'=>'',),
@@ -564,6 +602,7 @@
  'natural:scrub' => array('label'=>'Scrub','frequency'=>20,'icon'=>'',),
  'natural:fen' => array('label'=>'Fen','frequency'=>20,'icon'=>'',),
  'building:yes' => array('label'=>'Building','frequency'=>200,'icon'=>'',),
+ 'mountain_pass:yes' => array('label'=>'Mountain Pass','frequency'=>200,'icon'=>'',),
 
  'amenity:parking' => array('label'=>'Parking','frequency'=>3157,'icon'=>'',),
  'highway:bus_stop' => array('label'=>'Bus Stop','frequency'=>35777,'icon'=>'transport_bus_stop2',),
@@ -577,9 +616,10 @@
  'railway:disused_station' => array('label'=>'Disused Station','frequency'=>114,'icon'=>'',),
  'railway:abandoned' => array('label'=>'Abandoned','frequency'=>641,'icon'=>'',),
  'railway:disused' => array('label'=>'Disused','frequency'=>72,'icon'=>'',),
-			);		
+				);
 	}
-	
+
+
 	function getClassTypesWithImportance()
 	{
 		$aOrders = getClassTypes();
@@ -590,27 +630,31 @@
 		}
 		return $aOrders;
 	}
-	
-	
 
-    function javascript_renderData($xVal)
-    {
-        header("Access-Control-Allow-Origin: *");
 
-        $jsonout = json_encode($xVal);
+	function javascript_renderData($xVal)
+	{
+		header("Access-Control-Allow-Origin: *");
+		$jsonout = json_encode($xVal);
 
-		if( ! isset($_GET['json_callback'])) {
+		if( ! isset($_GET['json_callback']))
+		{
 			header("Content-Type: application/json; charset=UTF-8");
 			echo $jsonout;
-		} else {
-			if (preg_match('/^[$_\p{L}][$_\p{L}\p{Nd}.[\]]*$/u',$_GET['json_callback'])) {
+		} else
+		{
+			if (preg_match('/^[$_\p{L}][$_\p{L}\p{Nd}.[\]]*$/u',$_GET['json_callback']))
+			{
 				header("Content-Type: application/javascript; charset=UTF-8");
 				echo $_GET['json_callback'].'('.$jsonout.')';
-			} else {
+			}
+			else
+			{
 				header('HTTP/1.0 400 Bad Request');
 			}
 		}
-    }
+	}
+
 
 	function _debugDumpGroupedSearches($aData, $aTokens)
 	{
@@ -629,11 +673,11 @@
 			}
 		}
 		echo "<table border=\"1\">";
-		echo "<tr><th>rank</th><th>Name Tokens</th><th>Address Tokens</th><th>country</th><th>operator</th><th>class</th><th>type</th><th>house#</th><th>Lat</th><th>Lon</th><th>Radius</th></tr>";
+		echo "<tr><th>rank</th><th>Name Tokens</th><th>Name Not</th><th>Address Tokens</th><th>Address Not</th><th>country</th><th>operator</th><th>class</th><th>type</th><th>house#</th><th>Lat</th><th>Lon</th><th>Radius</th></tr>";
 		foreach($aData as $iRank => $aRankedSet)
 		{
 			foreach($aRankedSet as $aRow)
-			{		
+			{
 				echo "<tr>";
 				echo "<td>$iRank</td>";
 
@@ -646,6 +690,15 @@
 				}
 				echo "</td>";
 
+				echo "<td>";
+				$sSep = '';
+				foreach($aRow['aNameNonSearch'] as $iWordID)
+				{
+					echo $sSep.'#'.$aWordsIDs[$iWordID].'#';
+					$sSep = ', ';
+				}
+				echo "</td>";
+
 				echo "<td>";
 				$sSep = '';
 				foreach($aRow['aAddress'] as $iWordID)
@@ -655,6 +708,15 @@
 				}
 				echo "</td>";
 
+				echo "<td>";
+				$sSep = '';
+				foreach($aRow['aAddressNonSearch'] as $iWordID)
+				{
+					echo $sSep.'#'.$aWordsIDs[$iWordID].'#';
+					$sSep = ', ';
+				}
+				echo "</td>";
+
 				echo "<td>".$aRow['sCountryCode']."</td>";
 
 				echo "<td>".$aRow['sOperator']."</td>";
@@ -666,7 +728,7 @@
 				echo "<td>".$aRow['fLat']."</td>";
 				echo "<td>".$aRow['fLon']."</td>";
 				echo "<td>".$aRow['fRadius']."</td>";
-	
+
 				echo "</tr>";
 			}
 		}
@@ -677,120 +739,51 @@
 	function getAddressDetails(&$oDB, $sLanguagePrefArraySQL, $iPlaceID, $sCountryCode = false, $bRaw = false)
 	{
 		$sSQL = "select *,get_name_by_language(name,$sLanguagePrefArraySQL) as localname from get_addressdata($iPlaceID)";
-		IF (!$bRaw) $sSQL .= " WHERE isaddress OR type = 'country_code'";
+		if (!$bRaw) $sSQL .= " WHERE isaddress OR type = 'country_code'";
 		$sSQL .= " order by rank_address desc,isaddress desc";
-	        $aAddressLines = $oDB->getAll($sSQL);
-        	if (PEAR::IsError($aAddressLines))
-	        {
-        	        var_dump($aAddressLines);
-                	exit;
-	        }
+
+		$aAddressLines = $oDB->getAll($sSQL);
+		if (PEAR::IsError($aAddressLines))
+		{
+			var_dump($aAddressLines);
+			exit;
+		}
 		if ($bRaw) return $aAddressLines;
-//echo "<pre>";
-//var_dump($aAddressLines);
+		//echo "<pre>";
+		//var_dump($aAddressLines);
 		$aAddress = array();
+		$aFallback = array();
 		$aClassType = getClassTypes();
 		foreach($aAddressLines as $aLine)
 		{
+			$bFallback = false;
 			$aTypeLabel = false;
 			if (isset($aClassType[$aLine['class'].':'.$aLine['type'].':'.$aLine['admin_level']])) $aTypeLabel = $aClassType[$aLine['class'].':'.$aLine['type'].':'.$aLine['admin_level']];
 			elseif (isset($aClassType[$aLine['class'].':'.$aLine['type']])) $aTypeLabel = $aClassType[$aLine['class'].':'.$aLine['type']];
-			else $aTypeLabel = array('simplelabel'=>$aLine['class']);
-			if ($aTypeLabel && ($aLine['localname'] || $aLine['housenumber']))
+			elseif (isset($aClassType['boundary:administrative:'.((int)($aLine['rank_address']/2))]))
+			{
+				$aTypeLabel = $aClassType['boundary:administrative:'.((int)($aLine['rank_address']/2))];
+				$bFallback = true;
+			}
+			else
+			{
+				$aTypeLabel = array('simplelabel'=>'address'.$aLine['rank_address']);
+				$bFallback = true;
+			}
+			if ($aTypeLabel && ((isset($aLine['localname']) && $aLine['localname']) || (isset($aLine['housenumber']) && $aLine['housenumber'])))
 			{
 				$sTypeLabel = strtolower(isset($aTypeLabel['simplelabel'])?$aTypeLabel['simplelabel']:$aTypeLabel['label']);
 				$sTypeLabel = str_replace(' ','_',$sTypeLabel);
-				if (!isset($aAddress[$sTypeLabel]) && $aLine['localname']) $aAddress[$sTypeLabel] = $aLine['localname']?$aLine['localname']:$aLine['housenumber'];
-			}
-		}
-//var_dump($aAddress);
-//exit;
-		return $aAddress;
-
-		$aHouseNumber = $oDB->getRow('select housenumber, get_name_by_language(name,ARRAY[\'addr:housename\']) as housename,rank_search,postcode from placex where place_id = '.$iPlaceID);
-		$sHouseNumber = $aHouseNumber['housenumber'];
-		$sHouseName = $aHouseNumber['housename'];
-		$sPostcode = $aHouseNumber['postcode'];
-		$iRank = $aHouseNumber['rank_search'];
-
-	        // Address
-        	$sSQL = "select country_code, placex.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, rank_search, ";
-	        $sSQL .= "get_searchrank_label(rank_search) as rank_search_label, fromarea, isaddress, distance, ";
-        	$sSQL .= " CASE WHEN type = 'postcode' THEN postcode ELSE get_name_by_language(name,$sLanguagePrefArraySQL) END as localname, ";
-		$sSQL .= " length(name::text) as namelength ";
-	        $sSQL .= " from place_addressline join placex on (address_place_id = placex.place_id)";
-        	$sSQL .= " where place_addressline.place_id = $iPlaceID and (rank_address > 0 OR address_place_id = $iPlaceID)";
-		if (!$bRaw) $sSQL .= " and isaddress";
-        	$sSQL .= " order by cached_rank_address desc,isaddress desc,fromarea desc,distance asc,rank_search desc,namelength desc";
-//var_dump($sSQL);
-	        $aAddressLines = $oDB->getAll($sSQL);
-        	if (PEAR::IsError($aAddressLines))
-	        {
-        	        var_dump($aAddressLines);
-                	exit;
-	        }
-		if ($bRaw) return $aAddressLines;
-	
-		$aClassType = getClassTypes();
-
-		$iMinRank = 100;
-		$aAddress = array();
-		if ($iRank >= 28 && $sHouseNumber) $aAddress['house_number'] = $sHouseNumber;
-		if ($iRank >= 28 && $sHouseName) $aAddress['house_name'] = $sHouseName;
-		foreach($aAddressLines as $aLine)
-		{
-			if (!$sCountryCode) $sCountryCode = $aLine['country_code'];
-			if ($aLine['rank_address'] < $iMinRank)
-			{
-				$aTypeLabel = false;
-				if (isset($aClassType[$aLine['class'].':'.$aLine['type'].':'.$aLine['admin_level']])) $aTypeLabel = $aClassType[$aLine['class'].':'.$aLine['type'].':'.$aLine['admin_level']];
-				elseif (isset($aClassType[$aLine['class'].':'.$aLine['type']])) $aTypeLabel = $aClassType[$aLine['class'].':'.$aLine['type']];
-				else $aTypeLabel = array('simplelabel'=>$aLine['class']);
-				if ($aTypeLabel && ($aLine['localname'] || $aLine['housenumber']))
+				if (!isset($aAddress[$sTypeLabel]) || (isset($aFallback[$sTypeLabel]) && $aFallback[$sTypeLabel]))
 				{
-					$sTypeLabel = strtolower(isset($aTypeLabel['simplelabel'])?$aTypeLabel['simplelabel']:$aTypeLabel['label']);
-					$sTypeLabel = str_replace(' ','_',$sTypeLabel);
-					if (!isset($aAddress[$sTypeLabel]) && $aLine['localname']) $aAddress[$sTypeLabel] = $aLine['localname']?$aLine['localname']:$aLine['housenumber'];
+					$aAddress[$sTypeLabel] = $aLine['localname']?$aLine['localname']:$aLine['housenumber'];
 				}
-				$iMinRank = $aLine['rank_address'];
+				$aFallback[$sTypeLabel] = $bFallback;
 			}
 		}
-
-		if ($sPostcode)
-		{
-			$aAddress['postcode'] = $sPostcode;
-		}
-
-		if ($iMinRank > 4 && $sCountryCode)
-		{
-			$sSQL = "select get_name_by_language(country_name.name,$sLanguagePrefArraySQL) as name";
-			$sSQL .= " from country_name where country_code = '$sCountryCode'";
-			$sCountryName = $oDB->getOne($sSQL);
-			if ($sCountryName)
-			{
-				$aAddress['country'] = $sCountryName;
-			}
-		}
-		if ($sCountryCode)
-		{
-			$aAddress['country_code'] = $sCountryCode;
-		}
-
 		return $aAddress;
 	}
 
-	function getWordSuggestions(&$oDB, $sWord)
-	{
-		$sWordQuoted = getDBQuoted(trim($sWord));
-		$sSQL = "select *,levenshtein($sWordQuoted,word) from test_token ";
-		$sSQL .= "where (metaphone = dmetaphone($sWordQuoted) or metaphonealt = dmetaphone($sWordQuoted) or ";
-		$sSQL .= "metaphone = dmetaphone_alt($sWordQuoted) or metaphonealt = dmetaphone_alt($sWordQuoted)) ";
-		$sSQL .= "and len between length($sWordQuoted)-2 and length($sWordQuoted)+2 ";
-		$sSQL .= "and levenshtein($sWordQuoted,word) < 3 ";
-		$sSQL .= "order by levenshtein($sWordQuoted,word) asc, abs(len - length($sWordQuoted)) asc limit 20";
-		$aSimilar = $oDB->getAll($sSQL);
-		return $aSimilar;
-	}
 
 	function geocodeReverse($fLat, $fLon, $iZoom=18)
 	{
@@ -800,27 +793,27 @@
 
 		// Zoom to rank, this could probably be calculated but a lookup gives fine control
 		$aZoomRank = array(
-			0 => 2, // Continent / Sea
-			1 => 2,
-			2 => 2,
-			3 => 4, // Country
-			4 => 4,
-			5 => 8, // State
-			6 => 10, // Region
-			7 => 10, 
-			8 => 12, // County
-			9 => 12,  
-			10 => 17, // City
-			11 => 17, 
-			12 => 18, // Town / Village
-			13 => 18, 
-			14 => 22, // Suburb
-			15 => 22,
-			16 => 26, // Street, TODO: major street?
-			17 => 26, 
-			18 => 30, // or >, Building
-			19 => 30, // or >, Building
-			);
+				0 => 2, // Continent / Sea
+				1 => 2,
+				2 => 2,
+				3 => 4, // Country
+				4 => 4,
+				5 => 8, // State
+				6 => 10, // Region
+				7 => 10,
+				8 => 12, // County
+				9 => 12,
+				10 => 17, // City
+				11 => 17,
+				12 => 18, // Town / Village
+				13 => 18,
+				14 => 22, // Suburb
+				15 => 22,
+				16 => 26, // Street, TODO: major street?
+				17 => 26,
+				18 => 30, // or >, Building
+				19 => 30, // or >, Building
+				);
 		$iMaxRank = isset($aZoomRank[$iZoom])?$aZoomRank[$iZoom]:28;
 
 		// Find the nearest point
@@ -851,12 +844,12 @@
 			$sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') ';
 			$sSQL .= ' OR ST_DWithin('.$sPointSQL.', ST_Centroid(geometry), '.$fSearchDiam.'))';
 			$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1';
-//var_dump($sSQL);
+			//var_dump($sSQL);
 			$aPlace = $oDB->getRow($sSQL);
 			$iPlaceID = $aPlace['place_id'];
 			if (PEAR::IsError($iPlaceID))
 			{
-				var_Dump($sSQL, $iPlaceID); 
+				var_Dump($sSQL, $iPlaceID);
 				exit;
 			}
 		}
@@ -868,7 +861,7 @@
 			$iPlaceID = $oDB->getOne($sSQL);
 			if (PEAR::IsError($iPlaceID))
 			{
-				var_Dump($sSQL, $iPlaceID); 
+				var_Dump($sSQL, $iPlaceID);
 				exit;
 			}
 
@@ -878,7 +871,7 @@
 				$iPlaceID = $oDB->getOne($sSQL);
 				if (PEAR::IsError($iPlaceID))
 				{
-					var_Dump($sSQL, $iPlaceID); 
+					var_Dump($sSQL, $iPlaceID);
 					exit;
 				}
 			}
@@ -890,3 +883,8 @@
 
 		return $iPlaceID;
 	}
+
+	function addQuotes($s)
+	{
+		return "'".$s."'";
+	}

lib/log.php

@@ -13,7 +13,7 @@
 				$_SERVER["REMOTE_ADDR"],
 				$_SERVER['QUERY_STRING'],
 				$sQuery
-			);
+				);
 
 		if (CONST_Log_DB)
 		{
@@ -31,21 +31,21 @@
 
 		if (CONST_Log_File && CONST_Log_File_ReverseLog != '')
 		{
-                if ($sType == 'reverse')
-                {
-                        $aStartTime = explode('.',$hLog[0]);
-                        file_put_contents(CONST_Log_File_ReverseLog,
-                                $aStartTime[0].','.$aStartTime[1].','.
-                                php_uname('n').','.
-                                '"'.addslashes(isset($_SERVER['HTTP_REFERER'])?$_SERVER['HTTP_REFERER']:'').'",'.
-                                '"'.addslashes($hLog[1]).'",'.
-                                $_GET['lat'].','.
-                                $_GET['lon'].','.
-                                $_GET['zoom'].','.
-                                '"'.addslashes($_SERVER['HTTP_USER_AGENT']).'",'.
-                                '"'.addslashes($sOutputFormat).'"'."\n",
-                        FILE_APPEND);
-                }
+			if ($sType == 'reverse')
+			{
+				$aStartTime = explode('.',$hLog[0]);
+				file_put_contents(CONST_Log_File_ReverseLog,
+								$aStartTime[0].','.$aStartTime[1].','.
+								php_uname('n').','.
+								'"'.addslashes(isset($_SERVER['HTTP_REFERER'])?$_SERVER['HTTP_REFERER']:'').'",'.
+								'"'.addslashes($hLog[1]).'",'.
+								$_GET['lat'].','.
+								$_GET['lon'].','.
+								$_GET['zoom'].','.
+								'"'.addslashes($_SERVER['HTTP_USER_AGENT']).'",'.
+								'"'.addslashes($sOutputFormat).'"'."\n",
+								FILE_APPEND);
+			}
 		}
 
 		return $hLog;
@@ -63,7 +63,7 @@
 			$sSQL .= ' where starttime = '.getDBQuoted($hLog[0]);
 			$sSQL .= ' and ipaddress = '.getDBQuoted($hLog[1]);
 			$sSQL .= ' and query = '.getDBQuoted($hLog[3]);
-                	$oDB->query($sSQL);
+			$oDB->query($sSQL);
 
 			$sSQL = 'update new_query_log set endtime = '.getDBQuoted($sEndTime).', results = '.$iNumResults;
 			$sSQL .= ' where starttime = '.getDBQuoted($hLog[0]);
@@ -74,17 +74,17 @@
 
 		if (CONST_Log_File && CONST_Log_File_SearchLog != '')
 		{
-                $aStartTime = explode('.',$hLog[0]);
-                file_put_contents(CONST_Log_File_SearchLog,
-                                $aStartTime[0].','.$aStartTime[1].','.
-                                php_uname('n').','.
-                                '"'.addslashes(isset($_SERVER['HTTP_REFERER'])?$_SERVER['HTTP_REFERER']:'').'",'.
-                                '"'.addslashes($hLog[1]).'",'.
-                                '"'.addslashes($hLog[3]).'",'.
-                                '"'.addslashes($_SERVER['HTTP_USER_AGENT']).'",'.
-                                '"'.addslashes((isset($_GET['format']))?$_GET['format']:'').'",'.
-                                $iNumResults."\n",
-                        FILE_APPEND);
+			$aStartTime = explode('.',$hLog[0]);
+			file_put_contents(CONST_Log_File_SearchLog,
+					$aStartTime[0].','.$aStartTime[1].','.
+					php_uname('n').','.
+					'"'.addslashes(isset($_SERVER['HTTP_REFERER'])?$_SERVER['HTTP_REFERER']:'').'",'.
+					'"'.addslashes($hLog[1]).'",'.
+					'"'.addslashes($hLog[3]).'",'.
+					'"'.addslashes($_SERVER['HTTP_USER_AGENT']).'",'.
+					'"'.addslashes((isset($_GET['format']))?$_GET['format']:'').'",'.
+					$iNumResults."\n",
+					FILE_APPEND);
 		}
 
 	}

lib/template/address-json.php

@@ -3,7 +3,7 @@
 
 	if (!sizeof($aPlace))
 	{
-		if ($sError)
+		if (isset($sError))
 			$aFilteredPlaces['error'] = $sError;
 		else
 			$aFilteredPlaces['error'] = 'Unable to geocode';
@@ -21,7 +21,7 @@
                 if (isset($aPlace['lat'])) $aFilteredPlaces['lat'] = $aPlace['lat'];
                 if (isset($aPlace['lon'])) $aFilteredPlaces['lon'] = $aPlace['lon'];
 		$aFilteredPlaces['display_name'] = $aPlace['langaddress'];
-		if ($bShowAddressDetails) $aFilteredPlaces['address'] = $aAddress;
+		if ($bShowAddressDetails) $aFilteredPlaces['address'] = $aPlace['aAddress'];
 	}
 
 	javascript_renderData($aFilteredPlaces);

lib/template/address-jsonv2.php

@@ -3,7 +3,7 @@
 
 	if (!sizeof($aPlace))
 	{
-		if ($sError)
+		if (isset($sError))
 			$aFilteredPlaces['error'] = $sError;
 		else
 			$aFilteredPlaces['error'] = 'Unable to geocode';
@@ -26,7 +26,7 @@
 
 		$aFilteredPlaces['display_name'] = $aPlace['langaddress'];
                 $aFilteredPlaces['name'] = $aPlace['placename'];
-		if ($bShowAddressDetails && $aAddress && sizeof($aAddress)) $aFilteredPlaces['address'] = $aAddress;
+		if ($bShowAddressDetails && $aPlace['aAddress'] && sizeof($aPlace['aAddress'])) $aFilteredPlaces['address'] = $aPlace['aAddress'];
 	}
 
 	javascript_renderData($aFilteredPlaces);

lib/template/address-xml.php

@@ -1,6 +1,5 @@
 <?php
 	header("content-type: text/xml; charset=UTF-8");
-	header("Access-Control-Allow-Origin: *");
 
 	echo "<";
 	echo "?xml version=\"1.0\" encoding=\"UTF-8\" ?";
@@ -14,7 +13,7 @@
 
 	if (!sizeof($aPlace))
 	{
-		if ($sError)
+		if (isset($sError))
 			echo "<error>$sError</error>";
 		else
 			echo "<error>Unable to geocode</error>";
@@ -29,9 +28,9 @@
 		if (isset($aPlace['lon'])) echo ' lon="'.htmlspecialchars($aPlace['lon']).'"';
 		echo ">".htmlspecialchars($aPlace['langaddress'])."</result>";
 
-        if ($bShowAddressDetails) {
+		if ($bShowAddressDetails) {
 			echo "<addressparts>";
-			foreach($aAddress as $sKey => $sValue)
+			foreach($aPlace['aAddress'] as $sKey => $sValue)
 			{
 				$sKey = str_replace(' ','_',$sKey);
 				echo "<$sKey>";
@@ -39,7 +38,7 @@
 				echo "</$sKey>";
 			}
 			echo "</addressparts>";
-        }
+		}
 	}
-	
+
 	echo "</reversegeocode>";

lib/template/details-error-html.php

@@ -0,0 +1,180 @@
+<?php
+	header("content-type: text/html; charset=UTF-8");
+?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <title>OpenStreetMap Nominatim: <?php echo $aPointDetails['localname'];?></title>
+    <style>
+body {
+	margin:0px;
+	padding:16px;
+  background:#ffffff;
+  height: 100%;
+  font: normal 12px/15px arial,sans-serif;
+}
+.line{
+  margin-left:20px;
+}
+.name{
+  font-weight: bold;
+}
+.notused{
+  color:#ddd;
+}
+.noname{
+  color:#800;
+}
+#map {
+  width:500px;
+  height:500px;
+  border: 2px solid #666;
+  float: right;
+}
+    </style>
+	<script src="js/OpenLayers.js"></script>
+	<script src="js/tiles.js"></script>
+	<script type="text/javascript">
+        
+		var map;
+
+    function init() {
+			map = new OpenLayers.Map ("map", {
+                controls:[
+										new OpenLayers.Control.Permalink(),
+										new OpenLayers.Control.Navigation(),
+										new OpenLayers.Control.PanZoomBar(),
+										new OpenLayers.Control.MouseDefaults(),
+										new OpenLayers.Control.MousePosition(),
+										new OpenLayers.Control.Attribution()],
+                maxExtent: new OpenLayers.Bounds(-20037508.34,-20037508.34,20037508.34,20037508.34),
+                maxResolution: 156543.0399,
+                numZoomLevels: 19,
+                units: 'm',
+                projection: new OpenLayers.Projection("EPSG:900913"),
+                displayProjection: new OpenLayers.Projection("EPSG:4326")
+            	} );
+			map.addLayer(new OpenLayers.Layer.OSM.<?php echo CONST_Tile_Default;?>("Default"));
+
+                        var layer_style = OpenLayers.Util.extend({}, OpenLayers.Feature.Vector.style['default']);
+                        layer_style.fillOpacity = 0.2;
+                        layer_style.graphicOpacity = 0.2;
+
+			vectorLayer = new OpenLayers.Layer.Vector("Points", {style: layer_style});
+			map.addLayer(vectorLayer);
+
+            var proj_EPSG4326 = new OpenLayers.Projection("EPSG:4326");
+			var proj_map = map.getProjectionObject();
+
+            freader = new OpenLayers.Format.WKT({
+                'internalProjection': proj_map,
+                'externalProjection': proj_EPSG4326
+            });
+
+            var bounds;
+            <?php if ($aPointDetails['prevgeom']) { ?>
+            var feature = freader.read('<?php echo $aPointDetails['prevgeom'];?>');
+            if (feature) {
+                bounds = feature.geometry.getBounds();
+
+            }
+            feature.style = {
+				strokeColor: "#777777",
+				fillColor: "#F0F0F0",
+				strokeWidth: 2,
+				strokeOpacity: 0.75,
+				fillOpacity: 0.75,
+                strokeDashstyle: "longdash"
+			};
+            vectorLayer.addFeatures([feature]);
+            <?php } ?>
+
+            <?php if ($aPointDetails['newgeom']) { ?>
+            feature = freader.read('<?php echo $aPointDetails['newgeom'];?>');
+            if (feature) {
+                if (!bounds) {
+                    bounds = feature.geometry.getBounds();
+                } 
+                else 
+                {
+                    bounds.extend(feature.geometry.getBounds());
+                }
+            }
+            feature.style = {
+				strokeColor: "#75ADFF",
+				fillColor: "#FFF7F0",
+				strokeWidth: 2,
+				strokeOpacity: 0.75,
+				fillOpacity: 0.75
+			};
+            vectorLayer.addFeatures([feature]);
+            <?php } ?>
+
+            <?php if (isset($aPointDetails['error_x'])) { ?>
+            var pt = new OpenLayers.Geometry.Point(<?php echo $aPointDetails['error_x'].','.$aPointDetails['error_y'];?>);
+            pt = pt.transform(proj_EPSG4326, proj_map);
+            feature = new OpenLayers.Feature.Vector(pt, null,
+                       {
+                            graphicName : "x",
+                            fillColor: "#FF0000",
+                            graphic : true,
+                            pointRadius: 6
+                       });
+            vectorLayer.addFeatures([feature]);
+            <?php } ?>
+
+
+            map.zoomToExtent(bounds);
+
+		}
+		
+	</script>
+  </head>
+  <body onload="init();">
+    <div id="map"></div>
+	<h1><?php echo $aPointDetails['localname'] ?></h1>
+	<div class="locationdetails">
+	<div>Type: <span class="type"><?php echo $aPointDetails['class'].':'.$aPointDetails['type'];?></span></div>
+ 
+<?php
+    $sOSMType = ($aPointDetails['osm_type'] == 'N'?'node':($aPointDetails['osm_type'] == 'W'?'way':   ($aPointDetails['osm_type'] == 'R'?'relation':'')));
+    if ($sOSMType) echo ' <div>OSM: <span class="osm"><span class="label"></span>'.$sOSMType.' <a     href="http://www.openstreetmap.org/browse/'.$sOSMType.'/'.$aPointDetails['osm_id'].'">'.              $aPointDetails['osm_id'].'</a></span></div>';
+?>
+
+    <p>This object has an invalid geometry.</p>
+    <p><b>Details:</b> <?php
+
+$sVal = $aPointDetails['errormessage']?$aPointDetails['errormessage']:'&nbsp;';
+$sOSMType = ($aPointDetails['osm_type'] == 'N'?'node':($aPointDetails['osm_type'] == 'W'?'way':($aPointDetails['osm_type'] == 'R'?'relation':'')));
+$sOSMID = $aPointDetails['osm_id'];
+
+if (isset($aPointDetails['error_x']))
+{
+	$sLat = $aPointDetails['error_y'];
+	$sLon = $aPointDetails['error_x'];
+	echo "<a href=\"http://www.openstreetmap.org/?lat=".$sLat."&lon=".$sLon."&zoom=18&layers=M&".$sOSMType."=".$sOSMID."\">".$sVal."</a>";
+}
+else
+{
+	echo $sVal;
+}
+?>
+    <p><b>Edit:</b> in <?php
+if (isset($aPointDetails['error_x']))
+{
+	$fWidth = 0.0002;
+	echo " <a href=\"http://localhost:8111/load_and_zoom?left=".($sLon-$fWidth)."&right=".($sLon+$fWidth)."&top=".($sLat+$fWidth)."&bottom=".($sLat-$fWidth)."\" target=\"josm\">Remote Control (JOSM / Merkaartor)</a>";
+	echo " | <a href=\"http://www.openstreetmap.org/edit?editor=potlatch2&bbox=".($sLon-$fWidth).",".($sLat-$fWidth).",".($sLon+$fWidth).",".($sLat+$fWidth)."\" target=\"potlatch2\">Potlatch 2</a>";
+}
+else
+{
+	echo " <a href=\"http://localhost:8111/import?url=http://www.openstreetmap.org/api/0.6/".$sOSMType.'/'.$sOSMID."/full\" target=\"josm\">Remote Control (JOSM / Merkaartor)</a>";
+	// Should be better to load by object id - but this doesn't seem to zoom correctly
+	//echo " <a href=\"http://localhost:8111/load_object?new_layer=true&objects=".strtolower($aPointDetails['osm_type']).$sOSMID."\" target=\"josm\">Remote Control (JOSM / Merkaartor)</a>";
+}
+
+?></p>
+
+  </body>
+</html>
+

lib/template/details-html.php

@@ -1,40 +1,15 @@
 <?php
 	header("content-type: text/html; charset=UTF-8");
 ?>
-<html>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
   <head>
     <title>OpenStreetMap Nominatim: <?php echo $aPointDetails['localname'];?></title>
-    <style>
-body {
-	margin:0px;
-	padding:16px;
-  background:#ffffff;
-  height: 100%;
-  font: normal 12px/15px arial,sans-serif;
-}
-.line{
-  margin-left:20px;
-}
-.name{
-  font-weight: bold;
-}
-.notused{
-  color:#ddd;
-}
-.noname{
-  color:#800;
-}
-#map {
-  width:500px;
-  height:500px;
-  border: 2px solid #666;
-  float: right;
-}
-    </style>
-	<script src="js/OpenLayers.js"></script>
-	<script src="js/tiles.js"></script>
+    <link href="css/details.css" rel="stylesheet" type="text/css" />
+	<script src="js/OpenLayers.js" type="text/javascript"></script>
+	<script src="js/tiles.js" type="text/javascript"></script>
 	<script type="text/javascript">
-        
+
 		var map;
 
     function init() {
@@ -43,7 +18,6 @@ body {
 										new OpenLayers.Control.Permalink(),
 										new OpenLayers.Control.Navigation(),
 										new OpenLayers.Control.PanZoomBar(),
-										new OpenLayers.Control.MouseDefaults(),
 										new OpenLayers.Control.MousePosition(),
 										new OpenLayers.Control.Attribution()],
                 maxExtent: new OpenLayers.Bounds(-20037508.34,-20037508.34,20037508.34,20037508.34),
@@ -53,43 +27,51 @@ body {
                 projection: new OpenLayers.Projection("EPSG:900913"),
                 displayProjection: new OpenLayers.Projection("EPSG:4326")
             	} );
-			map.addLayer(new OpenLayers.Layer.OSM.<?php echo CONST_Tile_Default;?>("Default",
-                        { attribution : '© <a target="_parent" href="http://www.openstreetmap.org">OpenStreetMap</a> and contributors, under an <a target="_parent" href="http://www.openstreetmap.org/copyright">open license</a>'}));
+			map.addLayer(new OpenLayers.Layer.OSM.<?php echo CONST_Tile_Default;?>("Default"));
 
-                        var layer_style = OpenLayers.Util.extend({}, OpenLayers.Feature.Vector.style['default']);
-                        layer_style.fillOpacity = 0.2;
-                        layer_style.graphicOpacity = 0.2;
+			var layer_style = OpenLayers.Util.extend({}, OpenLayers.Feature.Vector.style['default']);
+			layer_style.fillOpacity = 0.2;
+			layer_style.graphicOpacity = 0.2;
 
 			vectorLayer = new OpenLayers.Layer.Vector("Points", {style: layer_style});
 			map.addLayer(vectorLayer);
 
-			var pointList = [];
-			var style = {
-				strokeColor: "#75ADFF",
-				fillColor: "#F0F7FF",
-				strokeWidth: 2,
-				strokeOpacity: 0.75,
-				fillOpacity: 0.75
-			};
 			var proj_EPSG4326 = new OpenLayers.Projection("EPSG:4326");
 			var proj_map = map.getProjectionObject();
-			var latlon;
-<?php
-if (isset($aPolyPoints))
-{
-	foreach($aPolyPoints as $aPolyPoint)
-	{
-		echo "                        pointList.push(new OpenLayers.Geometry.Point(".$aPolyPoint[1].",".$aPolyPoint[2]."));\n";
-	}
-}
-?>
-			var linearRing = new OpenLayers.Geometry.LinearRing(pointList).transform(proj_EPSG4326, proj_map);;
-			var polygonFeature = new OpenLayers.Feature.Vector(new OpenLayers.Geometry.Polygon([linearRing]),null,style);
-			vectorLayer.addFeatures([polygonFeature]);
 
-			map.zoomToExtent(new OpenLayers.Bounds(<?php echo $aPointPolygon['minlon']?>, <?php echo $aPointPolygon['minlat']?>, <?php echo $aPointPolygon['maxlon']?>, <?php echo $aPointPolygon['maxlat']?>).transform(proj_EPSG4326, proj_map));
+			freader = new OpenLayers.Format.WKT({
+				'internalProjection': proj_map,
+				'externalProjection': proj_EPSG4326
+			});
+
+			var feature = freader.read('<?php echo $aPointDetails['outlinestring'];?>');
+			var featureCentre = freader.read('POINT(<?php echo $aPointDetails['lon'];?> <?php echo $aPointDetails['lat'];?>)');
+			if (feature) {
+				map.zoomToExtent(feature.geometry.getBounds());
+				feature.style = {
+					strokeColor: "#75ADFF",
+					fillColor: "#F0F7FF",
+					strokeWidth: <?php echo ($aPointDetails['isarea']=='t'?'2':'5');?>,
+					strokeOpacity: 0.75,
+					fillOpacity: 0.75,
+					pointRadius: 50
+				};
+
+<?php if ($aPointDetails['isarea']=='t') {?>
+				featureCentre.style = {
+					strokeColor: "#008800",
+					fillColor: "#338833",
+					strokeWidth: <?php echo ($aPointDetails['isarea']=='t'?'2':'5');?>,
+					strokeOpacity: 0.75,
+					fillOpacity: 0.75,
+					pointRadius: 8
+				};
+				vectorLayer.addFeatures([feature,featureCentre]);
+<?php } else { ?>
+				vectorLayer.addFeatures([feature]);
+<?php } ?>
+			}
 		}
-		
 	</script>
   </head>
   <body onload="init();">
@@ -98,7 +80,7 @@ if (isset($aPolyPoints))
 	echo '<h1>';
 	if ($aPointDetails['icon'])
 	{
-		echo '<img style="float:right;margin-right:40px;" src="'.CONST_Website_BaseURL.'images/mapicons/'.$aPointDetails['icon'].'.n.32.png'.'">';
+		echo '<img style="float:right;margin-right:40px;" src="'.CONST_Website_BaseURL.'images/mapicons/'.$aPointDetails['icon'].'.n.32.png'.'" alt="'.$aPointDetails['icon'].'" />';
 	}
 	echo $aPointDetails['localname'].'</h1>';
 	echo '<div class="locationdetails">';
@@ -109,12 +91,14 @@ if (isset($aPolyPoints))
 	}
 	echo ' </div>';
 	echo ' <div>Type: <span class="type">'.$aPointDetails['class'].':'.$aPointDetails['type'].'</span></div>';
+	echo ' <div>Last Updated: <span class="type">'.$aPointDetails['indexed_date'].'</span></div>';
 	echo ' <div>Admin Level: <span class="adminlevel">'.$aPointDetails['admin_level'].'</span></div>';
 	echo ' <div>Rank: <span class="rankaddress">'.$aPointDetails['rank_search_label'].'</span></div>';
-	if ($aPointDetails['importance']) echo ' <div>Importance: <span class="rankaddress">'.$aPointDetails['importance'].'</span></div>';
+	if ($aPointDetails['calculated_importance']) echo ' <div>Importance: <span class="rankaddress">'.$aPointDetails['calculated_importance'].($aPointDetails['importance']?'':' (estimated)').'</span></div>';
 	echo ' <div>Coverage: <span class="area">'.($aPointDetails['isarea']=='t'?'Polygon':'Point').'</span></div>';
+	echo ' <div>Centre Point: <span class="area">'.$aPointDetails['lat'].','.$aPointDetails['lon'].'</span></div>';
 	$sOSMType = ($aPointDetails['osm_type'] == 'N'?'node':($aPointDetails['osm_type'] == 'W'?'way':($aPointDetails['osm_type'] == 'R'?'relation':'')));
-	if ($sOSMType) echo ' <div>OSM: <span class="osm"><span class="label"></span>'.$sOSMType.' <a href="http://www.openstreetmap.org/browse/'.$sOSMType.'/'.$aPointDetails['osm_id'].'">'.$aPointDetails['osm_id'].'</a></span></div>';
+	if ($sOSMType) echo ' <div>OSM: <span class="osm">'.$sOSMType.' <a href="http://www.openstreetmap.org/browse/'.$sOSMType.'/'.$aPointDetails['osm_id'].'">'.$aPointDetails['osm_id'].'</a></span></div>';
 	if ($aPointDetails['wikipedia'])
 	{
 		list($sWikipediaLanguage,$sWikipediaArticle) = explode(':',$aPointDetails['wikipedia']);
@@ -129,7 +113,7 @@ if (isset($aPolyPoints))
 	echo '</div>';
 
 	echo '<h2>Address</h2>';
-	echo '<div class=\"address\">';
+	echo '<div class="address">';
 	$iPrevRank = 1000000;
 	$sPrevLocalName = '';
 	foreach($aAddressLines as $aAddressLine)
@@ -145,9 +129,9 @@ if (isset($aPolyPoints))
 		echo '<span class="name">'.(trim($aAddressLine['localname'])?$aAddressLine['localname']:'<span class="noname">No Name</span>').'</span>';
 		echo ' (';
 		echo '<span class="type"><span class="label">Type: </span>'.$aAddressLine['class'].':'.$aAddressLine['type'].'</span>';
-		if ($sOSMType) echo ', <span class="osm"><span class="label"></span>'.$sOSMType.' <a href="http://www.openstreetmap.org/browse/'.$sOSMType.'/'.$aAddressLine['osm_id'].'">'.$aAddressLine['osm_id'].'</a></span>';
-		echo ', <span class="adminlevel">'.$aAddressLine['admin_level'].'</span>';
-		echo ', <span class="rankaddress">'.$aAddressLine['rank_search_label'].'</span>';
+		if ($sOSMType) echo ', <span class="osm">'.$sOSMType.' <a href="http://www.openstreetmap.org/browse/'.$sOSMType.'/'.$aAddressLine['osm_id'].'">'.$aAddressLine['osm_id'].'</a></span>';
+		if (isset($aAddressLine['admin_level'])) echo ', <span class="adminlevel">'.$aAddressLine['admin_level'].'</span>';
+		if (isset($aAddressLine['rank_search_label'])) echo ', <span class="rankaddress">'.$aAddressLine['rank_search_label'].'</span>';
 //		echo ', <span class="area">'.($aAddressLine['fromarea']=='t'?'Polygon':'Point').'</span>';
 		echo ', <span class="distance">'.$aAddressLine['distance'].'</span>';
 		echo ' <a href="details.php?place_id='.$aAddressLine['place_id'].'">GOTO</a>';
@@ -200,7 +184,7 @@ if (isset($aPolyPoints))
 
 	if (sizeof($aParentOfLines))
 	{
-		echo '<h2>Parent Of (named features only):</h2>';
+		echo '<h2>Parent Of:</h2>';
 
 		$aGroupedAddressLines = array();
 		foreach($aParentOfLines as $aAddressLine)
@@ -231,6 +215,9 @@ if (isset($aPolyPoints))
 			echo '</div>';
 		}
 		}
+		if (sizeof($aParentOfLines) >= 500) {
+			echo '<p>There are more child objects which are not shown.</p>';
+		}
 		echo '</div>';
 	}
 

lib/template/search-batch-json.php

@@ -0,0 +1,88 @@
+<?php
+
+	$aOutput = array();
+	$aOutput['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright";
+	$aOutput['batch'] = array();
+
+	foreach($aBatchResults as $aSearchResults)
+	{
+		if (!$aSearchResults) $aSearchResults = array();
+		$aFilteredPlaces = array();
+		foreach($aSearchResults as $iResNum => $aPointDetails)
+		{
+			$aPlace = array(
+					'place_id'=>$aPointDetails['place_id'],
+				  );
+
+			$sOSMType = ($aPointDetails['osm_type'] == 'N'?'node':($aPointDetails['osm_type'] == 'W'?'way':($aPointDetails['osm_type'] == 'R'?'relation':'')));
+			if ($sOSMType)
+			{
+				$aPlace['osm_type'] = $sOSMType;
+				$aPlace['osm_id'] = $aPointDetails['osm_id'];
+			}
+
+        	        if (isset($aPointDetails['aBoundingBox']))
+                	{
+				$aPlace['boundingbox'] = array(
+					$aPointDetails['aBoundingBox'][0],
+					$aPointDetails['aBoundingBox'][1],
+					$aPointDetails['aBoundingBox'][2],
+					$aPointDetails['aBoundingBox'][3]);
+
+				if (isset($aPointDetails['aPolyPoints']) && $bShowPolygons)
+				{
+					$aPlace['polygonpoints'] = $aPointDetails['aPolyPoints'];
+				}
+	                }
+
+			if (isset($aPointDetails['zoom']))
+			{
+				$aPlace['zoom'] = $aPointDetails['zoom'];
+			}
+
+			$aPlace['lat'] = $aPointDetails['lat'];
+			$aPlace['lon'] = $aPointDetails['lon'];
+			$aPlace['display_name'] = $aPointDetails['name'];
+			$aPlace['place_rank'] = $aPointDetails['rank_search'];
+
+			$aPlace['category'] = $aPointDetails['class'];
+			$aPlace['type'] = $aPointDetails['type'];
+
+			$aPlace['importance'] = $aPointDetails['importance'];
+
+			if (isset($aPointDetails['icon']))
+			{
+				$aPlace['icon'] = $aPointDetails['icon'];
+			}
+
+			if (isset($aPointDetails['address']) && sizeof($aPointDetails['address'])>0)
+			{
+				$aPlace['address'] = $aPointDetails['address'];
+        	        }
+
+	                if (isset($aPointDetails['asgeojson']))
+        	        {
+				$aPlace['geojson'] = json_decode($aPointDetails['asgeojson']);
+        	        }
+
+	                if (isset($aPointDetails['assvg']))
+        	        {
+				$aPlace['svg'] = $aPointDetails['assvg'];
+	                }
+
+	                if (isset($aPointDetails['astext']))
+        	        {
+                	        $aPlace['geotext'] = $aPointDetails['astext'];
+	                }
+
+        	        if (isset($aPointDetails['askml']))
+                	{
+                        	$aPlace['geokml'] = $aPointDetails['askml'];
+	                }
+
+			$aFilteredPlaces[] = $aPlace;
+		}
+		$aOutput['batch'][] = $aFilteredPlaces;
+	}
+
+	javascript_renderData($aOutput, array('geojson'));

lib/template/search-html.php

@@ -1,205 +1,40 @@
 <?php
 	header("content-type: text/html; charset=UTF-8");
 ?>
-<html>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 	<title>OpenStreetMap Nominatim: Search</title>
 
 	<base href="<?php echo CONST_Website_BaseURL;?>" />
 	<link href="nominatim.xml" rel="search" title="Nominatim Search" type="application/opensearchdescription+xml" />
+	<link href="css/search.css" rel="stylesheet" type="text/css" />
 
-	<script src="js/OpenLayers.js"></script>
-	<script src="js/tiles.js"></script>
-	<script src="js/prototype-1.6.0.3.js"></script>
-
-	<style>
-* {-moz-box-sizing: border-box;}
-body {
-  margin:0px;
-  padding:0px;
-  overflow: hidden;
-  background:#ffffff;
-  height: 100%;
-  font: normal 12px/15px arial,sans-serif;
-}
-#seachheader {
-  position:absolute;
-  z-index:5;
-  top:0px;
-  left:0px;
-  width:100%;
-  height:38px;
-  background:#F0F7FF;
-  border-bottom: 2px solid #75ADFF;
-}
-#q {
-  width:300px;
-}
-#seachheaderfade1, #seachheaderfade2, #seachheaderfade3, #seachheaderfade4{
-  position:absolute;
-  z-index:4;
-  top:0px;
-  left:0px;
-  width:100%;
-  opacity: 0.15;
-  filter: alpha(opacity = 15);
-  background:#000000;
-  border: 1px solid #000000;
-}
-#seachheaderfade1{
-  height:39px;
-}
-#seachheaderfade2{
-  height:40px;
-}
-#seachheaderfade3{
-  height:41px;
-}
-#seachheaderfade4{
-  height:42px;
-}
-#searchresultsfade1, #searchresultsfade2, #searchresultsfade3, #searchresultsfade4 {
-  position:absolute;
-  z-index:2;
-  top:0px;
-  left:200px;
-  height: 100%;
-  opacity: 0.2;
-  filter: alpha(opacity = 20);
-  background:#ffffff;
-  border: 1px solid #ffffff;
-}
-#searchresultsfade1{
-  width:1px;
-}
-#searchresultsfade2{
-  width:2px;
-}
-#searchresultsfade3{
-  width:3px;
-}
-#searchresultsfade4{
-  width:4px;
-}
-
-#searchresults{
-  position:absolute;
-  z-index:3;
-  top:41px;
-  width:200px;
-  height: 100%;
-  background:#ffffff;
-  border: 1px solid #ffffff;
-  overflow: auto;
-}
-#map{
-  position:absolute;
-  z-index:1;
-  top:38px;
-  left:200px;
-  width:100%;
-  height:100%;
-  background:#eee;
-}
-#report{
-  position:absolute;
-  z-index:2;
-  top:38px;
-  left:200px;
-  width:100%;
-  height:100%;
-  background:#eee;
-  font: normal 12px/15px arial,sans-serif;
-  padding:20px;
-}
-#report table {
-  margin-left:20px;
-}
-#report th {
-  vertical-align:top;
-  text-align:left;
-}
-#report td.button {
-  text-align:right;
-}
-.result {
-  margin:5px;
-  margin-bottom:0px;
-  padding:2px;
-  padding-left:4px;
-  padding-right:4px;
-  border-radius: 5px;
-  -moz-border-radius: 5px;
-  -webkit-border-radius: 5px;
-  background:#F0F7FF;
-  border: 2px solid #D7E7FF;
-  font: normal 12px/15px arial,sans-serif;
-  cursor:pointer;
-}
-.result img{
-  float:right;
-}
-.result .latlon{
-  display: none;
-}
-.result .place_id{
-  display: none;
-}
-.result .type{
-  color: #999;
-  text-align:center;
-  font: normal 9px/10px arial,sans-serif;
-  padding-top:4px;
-}
-.result .details, .result .details a{
-  color: #999;
-  text-align:center;
-  font: normal 9px/10px arial,sans-serif;
-  padding-top:4px;
-}
-.noresults{
-  color: #000;
-  text-align:center;
-  font: normal 12px arial,sans-serif;
-  padding-top:4px;
-}
-.more{
-  color: #ccc;
-  text-align:center;
-  padding-top:4px;
-}
-.disclaimer{
-  color: #ccc;
-  text-align:center;
-  font: normal 9px/10px arial,sans-serif;
-  padding-top:4px;
-}
-form{
-  margin:0px;
-  padding:0px;
-}
-	</style>
+	<script src="js/OpenLayers.js" type="text/javascript"></script>
+	<script src="js/tiles.js" type="text/javascript"></script>
+	<script src="js/prototype-1.6.0.3.js" type="text/javascript"></script>
 
 	<script type="text/javascript">
-        
+
 		var map;
 
 		function handleResize()
 		{
 			if ($('searchresults'))
 			{
-				$('map').style.width = (document.documentElement.clientWidth > 0?document.documentElement.clientWidth:document.documentElement.offsetWidth) - 200;
-				$('report').style.width = (document.documentElement.clientWidth > 0?document.documentElement.clientWidth:document.documentElement.offsetWidth) - 200;
+				var viewwidth = ((document.documentElement.clientWidth > 0?document.documentElement.clientWidth:document.documentElement.offsetWidth) - 200) + 'px';
+				$('map').style.width = viewwidth;
+				$('report').style.width = viewwidth;
 			}
 			else
 			{
-				$('map').style.width = (document.documentElement.clientWidth > 0?document.documentElement.clientWidth:document.documentElement.offsetWidth) - 0;
-				$('map').style.left = 0;
+				$('map').style.width = ((document.documentElement.clientWidth > 0?document.documentElement.clientWidth:document.documentElement.offsetWidth) - 0) + 'px';
+				$('map').style.left = '0px';
 			}
-			
-			if ($('map')) $('map').style.height = (document.documentElement.clientHeight > 0?document.documentElement.clientHeight:document.documentElement.offsetHeight) - 38;
-			if ($('searchresults')) $('searchresults').style.height = (document.documentElement.clientHeight > 0?document.documentElement.clientHeight:document.documentElement.offsetHeight) - 38;
-			if ($('report')) $('report').style.height = (document.documentElement.clientHeight > 0?document.documentElement.clientHeight:document.documentElement.offsetHeight) - 38;
+
+			if ($('map')) $('map').style.height = ((document.documentElement.clientHeight > 0?document.documentElement.clientHeight:document.documentElement.offsetHeight) - 38) + 'px';
+			if ($('searchresults')) $('searchresults').style.height = ((document.documentElement.clientHeight > 0?document.documentElement.clientHeight:document.documentElement.offsetHeight) - 38) + 'px';
+			if ($('report')) $('report').style.height = ((document.documentElement.clientHeight > 0?document.documentElement.clientHeight:document.documentElement.offsetHeight) - 38) + 'px';
 		}
 		window.onresize = handleResize;
 
@@ -216,40 +51,34 @@ form{
 				map.panTo(lonLat, 10);
 		}
 
-		function panToLatLonBoundingBox(lat,lon,minlat,maxlat,minlon,maxlon,points) {
-		        var proj_EPSG4326 = new OpenLayers.Projection("EPSG:4326");
-		        var proj_map = map.getProjectionObject();
-                        map.zoomToExtent(new OpenLayers.Bounds(minlon,minlat,maxlon,maxlat).transform(proj_EPSG4326, proj_map));
-			var lonLat = new OpenLayers.LonLat(lon, lat).transform(new OpenLayers.Projection("EPSG:4326"), map.getProjectionObject());
+		function panToLatLonBoundingBox(lat,lon,minlat,maxlat,minlon,maxlon,wkt) {
+			vectorLayer.destroyFeatures();
+			var proj_EPSG4326 = new OpenLayers.Projection("EPSG:4326");
+			var proj_map = map.getProjectionObject();
+			map.zoomToExtent(new OpenLayers.Bounds(minlon,minlat,maxlon,maxlat).transform(proj_EPSG4326, proj_map));
+			var lonLat = new OpenLayers.LonLat(lon, lat).transform(proj_EPSG4326, proj_map);
 			map.panTo(lonLat, <?php echo $iZoom ?>);
 
-                        var pointList = [];
-                        var style = {
-                                strokeColor: "#75ADFF",
-                                fillColor: "#F0F7FF",
-                                strokeWidth: 2,
-                                strokeOpacity: 0.75,
-                                fillOpacity: 0.75
-                        };
-                        var proj_EPSG4326 = new OpenLayers.Projection("EPSG:4326");
-                        var proj_map = map.getProjectionObject();
-			if (points)
+			if (wkt)
 			{
-				points.each(function(p){
-					pointList.push(new OpenLayers.Geometry.Point(p[0],p[1]));
-					});
-        	                var linearRing = new OpenLayers.Geometry.LinearRing(pointList).transform(proj_EPSG4326, proj_map);;
-                	        var polygonFeature = new OpenLayers.Feature.Vector(new OpenLayers.Geometry.Polygon([linearRing]),null,style);
-				vectorLayer.destroyFeatures();
-                        	vectorLayer.addFeatures([polygonFeature]);
-			}
-			else
-			{
-				var lonLat = new OpenLayers.LonLat(lon, lat).transform(new OpenLayers.Projection("EPSG:4326"), map.getProjectionObject());
-				var point = new OpenLayers.Geometry.Point(lonLat.lon, lonLat.lat);
-				var pointFeature = new OpenLayers.Feature.Vector(point,null,style);
-				vectorLayer.destroyFeatures();
-				vectorLayer.addFeatures([pointFeature]);
+				var freader = new OpenLayers.Format.WKT({
+						'internalProjection': proj_map,
+						'externalProjection': proj_EPSG4326
+						});
+
+				var feature = freader.read(wkt);
+				if (feature)
+				{
+					feature.style = {
+							strokeColor: "#75ADFF",
+							fillColor: "#F0F7FF",
+							strokeWidth: 2,
+							strokeOpacity: 0.75,
+							fillOpacity: 0.75,
+							pointRadius: 100
+                    };
+					vectorLayer.addFeatures([feature]);
+				}
 			}
 		}
 
@@ -282,7 +111,6 @@ form{
                 controls:[
 										new OpenLayers.Control.Navigation(),
 										new OpenLayers.Control.PanZoomBar(),
-										new OpenLayers.Control.MouseDefaults(),
 										new OpenLayers.Control.MousePosition(),
 										new OpenLayers.Control.Attribution()],
                 maxExtent: new OpenLayers.Bounds(-20037508.34,-20037508.34,20037508.34,20037508.34),
@@ -295,8 +123,7 @@ form{
 									"moveend": mapEventMove
 								}
             	} );
-			map.addLayer(new OpenLayers.Layer.OSM.<?php echo CONST_Tile_Default;?>("Default",
-                         { attribution : '© <a target="_parent" href="http://www.openstreetmap.org">OpenStreetMap</a> and contributors, under an <a target="_parent" href="http://www.openstreetmap.org/copyright">open license</a>'}));
+			map.addLayer(new OpenLayers.Layer.OSM.<?php echo CONST_Tile_Default;?>("Default"));
 
 			var layer_style = OpenLayers.Util.extend({}, OpenLayers.Feature.Vector.style['default']);
 			layer_style.fillOpacity = 0.2;
@@ -321,26 +148,26 @@ form{
 
 	<div id="seachheader">
 		<form accept-charset="UTF-8" action="<?php echo CONST_Website_BaseURL; ?>search.php" method="get">
-			<table border="0" width="100%">
+			<table border="0" width="100%" summary="header">
 				<tr>
-					<td valign="center" style="width:30px;"><img src="images/logo.gif"></td>
-					<td valign="center" style="width:400px;"><input id="q" name="q" value="<?php echo htmlspecialchars($sQuery); 
-?>" style="width:270px;"><input type="text" id="viewbox" style="width:130px;" name="viewbox"></td>
-					<td style="width:80px;"><input type="submit" value="Search"></td>
-<?php if (CONST_Search_AreaPolygons) { ?>					<td style="width:100px;"><input type="checkbox" value="1" name="polygon" <?php if ($bShowPolygons) echo "checked"; ?>> Highlight</td>
+					<td valign="middle" style="width:30px;"><img alt="logo" src="images/logo.gif" /></td>
+					<td valign="middle" style="width:400px;"><input id="q" name="q" value="<?php echo htmlspecialchars($sQuery); 
+?>" style="width:270px;" /><input type="text" id="viewbox" style="width:120px;" name="viewbox" /></td>
+					<td style="width:80px;"><input type="submit" value="Search"/></td>
+<?php if (CONST_Search_AreaPolygons) { ?>					<td style="width:100px;"><input type="checkbox" value="1" name="polygon" <?php if ($bAsText) echo "checked='checked'"; ?>/> Highlight</td>
 <td style="text-align:right;">Data: <?php echo $sDataDate; ?></td>
 <td style="text-align:right;">
 <a href="http://wiki.openstreetmap.org/wiki/Nominatim" target="_blank">Documentation</a> | <a href="http://wiki.openstreetmap.org/wiki/Nominatim/FAQ" 
 target="_blank">FAQ</a></td>
 
-<?php } ?>					<td style="text-align:right;"><?php if ($sQuery) { ?><input type="button" value="Report Problem With Results" onclick="$('report').style.visibility=($('report').style.visibility=='hidden'?'visible':'hidden')"><?php } ?></td>
+<?php } ?>					<td style="text-align:right;"><?php if ($sQuery) { ?><input type="button" value="Report Problem With Results" onclick="$('report').style.visibility=($('report').style.visibility=='hidden'?'visible':'hidden')"/><?php } ?></td>
 				</tr>
 			</table>
 		</form>
 	</div>
 
 <?php
-	if ($sQuery || sizeof($aSearchResults))
+	if ($sQuery)
 	{
 ?>
 	<div id="searchresultsfade1"></div><div id="searchresultsfade2"></div><div id="searchresultsfade3"></div><div id="searchresultsfade4"></div>
@@ -359,8 +186,8 @@ target="_blank">FAQ</a></td>
 			echo ', '.$aResult['aBoundingBox'][1];
 			echo ', '.$aResult['aBoundingBox'][2];
 			echo ', '.$aResult['aBoundingBox'][3];
-			if (isset($aResult['aPolyPoints'])) echo ', '.json_encode($aResult['aPolyPoints']);
-			echo ');\'>';
+			if (isset($aResult['astext'])) echo ', "'.$aResult['astext'].'"';
+			echo ");'>\n";
 		}
 		elseif (isset($aResult['zoom']))
 		{
@@ -371,7 +198,7 @@ target="_blank">FAQ</a></td>
 			echo '<div class="result" onClick="panToLatLon('.$aResult['lat'].', '.$aResult['lon'].');">';
 		}
 
-		echo (isset($aResult['icon'])?'<img src="'.$aResult['icon'].'">':'');
+		echo (isset($aResult['icon'])?'<img alt="icon" src="'.$aResult['icon'].'"/>':'');
 		echo ' <span class="name">'.$aResult['name'].'</span>';
 		echo ' <span class="latlon">'.round($aResult['lat'],3).','.round($aResult['lon'],3).'</span>';
 		echo ' <span class="place_id">'.$aResult['place_id'].'</span>';
@@ -386,7 +213,7 @@ target="_blank">FAQ</a></td>
 	{
 		if ($sMoreURL)
 		{
-			echo '<div class="more"><a href="'.$sMoreURL.'">Search for more results</a></div>';
+			echo '<div class="more"><a href="'.htmlentities($sMoreURL).'">Search for more results</a></div>';
 		}
 	}
 	else
@@ -396,7 +223,7 @@ target="_blank">FAQ</a></td>
 
 ?>
 		<div class="disclaimer">Addresses and postcodes are approximate
-			<input type="button" value="Report Problem" onclick="$('report').style.visibility=($('report').style.visibility=='hidden'?'visible':'hidden')">
+			<input type="button" value="Report Problem" onclick="$('report').style.visibility=($('report').style.visibility=='hidden'?'visible':'hidden')"/>
 		</div>
 	</div>
 <?php
@@ -411,12 +238,12 @@ href="http://wiki.openstreetmap.org/wiki/Nominatim/FAQ">FAQ</a>.  If your proble
 to check how the address was generated before reporting a problem.</p>
 		<p>Please use <a href="http://trac.openstreetmap.org/newticket?component=nominatim">trac.openstreetmap.org</a> to report problems 
 making sure to set 
-the component to 'nominatim'.  You can search for existing bug reports <a href="http://trac.openstreetmap.org/query?status=new&status=assigned&status=reopened&component=nominatim&order=priority">here</a>.</p>
+the component to 'nominatim'.  You can search for existing bug reports <a href="http://trac.openstreetmap.org/query?status=new&amp;status=assigned&amp;status=reopened&amp;component=nominatim&amp;order=priority">here</a>.</p>
 		<p>Please ensure that you include a full description of the problem, including the search query that you used, the problem with the result and, if 
 the problem relates to missing data, the osm id of the item that is missing.  Problems that contain enough detail are likely to get looked at before ones that 
 require significant research!</p>
 		</div>
-		
+
 <!--
  		<p>Please use this form to report problems with the search results.  Of particular interest are items missing, but please also use this form to 
 report any other problems.</p>
@@ -451,11 +278,7 @@ init();
 			echo ', '.$aResult['aBoundingBox'][1];
 			echo ', '.$aResult['aBoundingBox'][2];
 			echo ', '.$aResult['aBoundingBox'][3];
-			if (isset($aResult['aPolyPoints']))
-			{
-				echo ', ';
-				echo javascript_renderData($aResult['aPolyPoints']);
-			}
+			if (isset($aResult['astext'])) echo ", '".$aResult['astext']."'";
 			echo ');'."\n";
 		}
 		else

lib/template/search-json.php

@@ -1,4 +1,6 @@
 <?php
+	header("content-type: application/json; charset=UTF-8");
+
 	$aFilteredPlaces = array();
 	foreach($aSearchResults as $iResNum => $aPointDetails)
 	{
@@ -39,7 +41,10 @@
 
 		$aPlace['class'] = $aPointDetails['class'];
 		$aPlace['type'] = $aPointDetails['type'];
-		if ($aPointDetails['icon'])
+
+		$aPlace['importance'] = $aPointDetails['importance'];
+
+		if (isset($aPointDetails['icon']) && $aPointDetails['icon'])
 		{
 			$aPlace['icon'] = $aPointDetails['icon'];
 		}
@@ -49,6 +54,26 @@
 			$aPlace['address'] = $aPointDetails['address'];
                 }
 
+                if (isset($aPointDetails['asgeojson']))
+                {
+                        $aPlace['geojson'] = json_decode($aPointDetails['asgeojson']);
+                }
+
+                if (isset($aPointDetails['assvg']))
+                {
+                        $aPlace['svg'] = $aPointDetails['assvg'];
+                }
+
+                if (isset($aPointDetails['astext']))
+                {
+                        $aPlace['geotext'] = $aPointDetails['astext'];
+                }
+
+		if (isset($aPointDetails['askml']))
+		{
+			$aPlace['geokml'] = $aPointDetails['askml'];
+		}
+
 		$aFilteredPlaces[] = $aPlace;
 	}
 

lib/template/search-jsonv2.php

@@ -36,11 +36,14 @@
 		$aPlace['lat'] = $aPointDetails['lat'];
 		$aPlace['lon'] = $aPointDetails['lon'];
 		$aPlace['display_name'] = $aPointDetails['name'];
-		$aPlace['place_rank'] = $aResult['rank_search'];
+		$aPlace['place_rank'] = $aPointDetails['rank_search'];
 
 		$aPlace['category'] = $aPointDetails['class'];
 		$aPlace['type'] = $aPointDetails['type'];
-		if ($aPointDetails['icon'])
+
+		$aPlace['importance'] = $aPointDetails['importance'];
+
+		if (isset($aPointDetails['icon']))
 		{
 			$aPlace['icon'] = $aPointDetails['icon'];
 		}
@@ -50,7 +53,27 @@
 			$aPlace['address'] = $aPointDetails['address'];
                 }
 
+                if (isset($aPointDetails['asgeojson']))
+                {
+			$aPlace['geojson'] = json_decode($aPointDetails['asgeojson']);
+                }
+
+                if (isset($aPointDetails['assvg']))
+                {
+			$aPlace['svg'] = $aPointDetails['assvg'];
+                }
+
+                if (isset($aPointDetails['astext']))
+                {
+                        $aPlace['geotext'] = $aPointDetails['astext'];
+                }
+
+                if (isset($aPointDetails['askml']))
+                {
+                        $aPlace['geokml'] = $aPointDetails['askml'];
+                }
+
 		$aFilteredPlaces[] = $aPlace;
 	}
 
-	javascript_renderData($aFilteredPlaces);
+	javascript_renderData($aFilteredPlaces, array('geojson'));

lib/template/search-xml.php

@@ -1,6 +1,5 @@
 <?php
 	header("content-type: text/xml; charset=UTF-8");
-	header("Access-Control-Allow-Origin: *");
 
 	echo "<";
 	echo "?xml version=\"1.0\" encoding=\"UTF-8\" ?";
@@ -10,7 +9,7 @@
 	echo " timestamp='".date(DATE_RFC822)."'";
 	echo " attribution='Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright'";
 	echo " querystring='".htmlspecialchars($sQuery, ENT_QUOTES)."'";
-	if (isset($sViewBox)) echo " viewbox='".htmlspecialchars($sViewBox, ENT_QUOTES)."'";
+	if ($sViewBox) echo " viewbox='".htmlspecialchars($sViewBox, ENT_QUOTES)."'";
 	echo " polygon='".($bShowPolygons?'true':'false')."'";
 	if (sizeof($aExcludePlaceIDs))
 	{
@@ -50,6 +49,27 @@
 			}
 		}
 
+		if (isset($aResult['asgeojson']))
+		{
+			echo ' geojson=\'';
+			echo $aResult['asgeojson'];
+			echo '\'';
+		}
+
+		if (isset($aResult['assvg']))
+		{
+			echo ' geosvg=\'';
+			echo $aResult['assvg'];
+			echo '\'';
+		}
+
+		if (isset($aResult['astext']))
+		{
+			echo ' geotext=\'';
+			echo $aResult['astext'];
+			echo '\'';
+		}
+
 		if (isset($aResult['zoom']))
 		{
 			echo " zoom='".$aResult['zoom']."'";
@@ -61,14 +81,27 @@
 
 		echo " class='".htmlspecialchars($aResult['class'])."'";
 		echo " type='".htmlspecialchars($aResult['type'])."'";
-		if ($aResult['icon'])
+		echo " importance='".htmlspecialchars($aResult['importance'])."'";
+		if (isset($aResult['icon']) && $aResult['icon'])
 		{
 			echo " icon='".htmlspecialchars($aResult['icon'], ENT_QUOTES)."'";
 		}
 
-		if (isset($aResult['address']))
+		if (isset($aResult['address']) || isset($aResult['askml']))
 		{
 			echo ">";
+		}
+
+		if (isset($aResult['askml']))
+		{
+			echo "\n<geokml>";
+			echo $aResult['askml'];
+			echo "</geokml>";
+		}
+
+		if (isset($aResult['address']))
+		{
+			echo "\n";
 			foreach($aResult['address'] as $sKey => $sValue)
 			{
 				$sKey = str_replace(' ','_',$sKey);
@@ -76,7 +109,10 @@
 				echo htmlspecialchars($sValue);
 				echo "</$sKey>";
 			}
+		}
 
+		if (isset($aResult['address']) || isset($aResult['askml']))
+		{
 			echo "</place>";
 		}
 		else

module/Makefile.in

@@ -1,5 +1,6 @@
 MODULES = nominatim
 PGXS := @POSTGRESQL_PGXS@
+PG_CONFIG := @PG_CONFIG@
 include $(PGXS)
 
 all:

module/nominatim.c

@@ -143,12 +143,12 @@ transliteration( PG_FUNCTION_ARGS )
 				resultdata++;
 			}
 		}
-		else
+		/*else
 		{
 			ereport( WARNING, ( errcode( ERRCODE_SUCCESSFUL_COMPLETION ),
 		              errmsg( "missing char: %i\n", *wchardata )));
 			
-		}
+		}*/
 		wchardata++;
 	}
 

munin/nominatim_importlag

@@ -2,6 +2,9 @@
 #
 # Plugin to monitor the age of the imported data in the rendering db
 #
+# Can be configured through libpq environment variables, for example
+# PGUSER, PGDATABASE, etc. See man page of psql for more information
+#
 # Parameters: 
 #
 #       config   (required)
@@ -21,7 +24,7 @@ if [ "$1" = "config" ]; then
 fi
  
 
-delay=`psql -d nominatim -c "copy (select extract(epoch from timezone('utc', now())-lastimportdate)::int from import_status) to stdout"`
+delay=`psql -c "copy (select extract(epoch from timezone('utc', now())-lastimportdate)::int from import_status) to stdout"`
  
  
 echo "age.value $delay"

munin/nominatim_query_speed

@@ -0,0 +1,121 @@
+#!/usr/bin/perl -w
+# Plugin to monitor response time of search queries.
+#
+# Based on a plugin by Dalibo <cedric.villemain@dalibo.com> 2007
+# Based on a plugin (postgres_block_read_) from Bj<42>rn Ruberg <bjorn@linpro.no>
+#
+# Licenced under GPL v2.
+#
+# Usage:
+#
+#       If required, give username, password and/or Postgresql server
+#       host through environment variables. See man page of psql for
+#       more info.
+#
+# Parameters:
+#
+#       config   (required)
+#
+# Magic markers
+#%# family=auto
+#%# capabilities=autoconf
+
+use strict;
+use DBI;
+use vars qw ( $debug $configure  );
+use constant _PGMINI => 70400;
+
+my $dbhost = $ENV{'PGHOST'} || '';
+my $dbname = $ENV{'PGDATABASE'} || 'nominatim';
+my $dbuser = $ENV{'PGUSER'} || 'postgres';
+my $dbport = $ENV{'PGPORT'} || '5432';
+
+my $dsn = "DBI:Pg:dbname=$dbname";
+$dsn   .=";host=$dbhost;port=$dbport" if $dbhost;
+my $pg_server_version;
+
+if (exists $ARGV[0]) {
+  if ($ARGV[0] eq 'autoconf') {
+    # Check for DBD::Pg
+    if (! eval "require DBD::Pg;") {
+      print "no (DBD::Pg not found)";
+      exit 1;
+    }
+    my $dbh = DBI->connect ($dsn,
+                            $dbuser,
+                            '',
+                            {RaiseError =>1});
+    if ($dbh) {
+      $pg_server_version = $dbh->{'pg_server_version'};
+      if ($pg_server_version < (_PGMINI)) {
+        $pg_server_version =~ /(\d)(\d){2,2}(\d){2,2}/;
+        print "PostgreSQL Server version " . (_PGMINI) . " or above is needed. Current is $1.$2.$3 \n";
+				exit 1;
+      }
+      print "yes\n";
+      exit 0;
+    } else {
+      print "no Unable to access Database $dbname on host $dbhost as user $dbuser.\nError returned was: ". $DBI::errstr;
+      exit 1;
+    }
+  } elsif ($ARGV[0] eq 'debug') {
+    # Set debug flag
+    $debug = 1;
+  } elsif ($ARGV[0] eq 'config') {
+    # Set config flag
+    $configure = 1;
+  }
+}
+
+print "# $dsn\n" if $debug;
+my $dbh = DBI->connect ($dsn,
+                        $dbuser,
+                        '',
+                        {RaiseError =>1});
+
+die ("no Unable to access Database $dbname on host $dbhost as user $dbuser.\nError returned was: ". $DBI::errstr."\n") unless($dbh);
+$pg_server_version = $dbh->{'pg_server_version'};
+
+if ($configure) {
+  print "graph_title Total Nominatim response time\n";
+  print "graph_vlabel Time to response\n";
+  print "graph_category Nominatim \n";
+  print "graph_period minute\n";
+  print "graph_args --base 1000\n";
+
+  print "avg.label Average time to response\n";
+  print "avg.draw LINE\n";
+  print "avg.type GAUGE\n";
+  print "avg.min 0\n";
+  print "avg.info Moving 5 minute average time to perform search\n";
+  print "avg.label Average time to response\n";
+
+  print "min.label Fastest time to response\n";
+  print "min.draw LINE\n";
+  print "min.type GAUGE\n";
+  print "min.min 0\n";
+  print "min.info Fastest query in last 5 minutes\n";
+
+  print "max.label Slowest time to response\n";
+  print "max.draw LINE\n";
+  print "max.type GAUGE\n";
+  print "max.min 0\n";
+  print "max.info Slowest query in last 5 minutes\n";
+
+} else {
+
+  my $sql = "select TO_CHAR(avg(endtime-starttime),'SS.MS'),TO_CHAR(min(endtime-starttime),'SS.MS'),TO_CHAR(max(endtime-starttime),'SS.MS') from new_query_log where starttime > 'now'::timestamp - '5 minutes'::interval";
+  print "# $sql\n" if $debug;
+  my $sth = $dbh->prepare($sql);
+  $sth->execute();
+  printf ("# Rows: %d\n",  $sth->rows) if $debug;
+  if ($sth->rows > 0) {
+    my ($avg, $min, $max) = $sth->fetchrow_array();
+    print "avg.value $avg\n";
+    print "min.value $min\n";
+    print "max.value $max\n";
+  }
+}
+
+exit 0;
+

munin/nominatim_requests

@@ -2,6 +2,9 @@
 #
 # Plugin to monitor the types of requsts made to the API
 #
+# Can be configured through libpq environment variables, for example
+# PGUSER, PGDATABASE, etc. See man page of psql for more information.
+#
 # Parameters: 
 #
 #       config   (required)
@@ -32,10 +35,10 @@ fi
 
 query="select count(*)/5.0 from new_query_log where starttime > (now() - interval '5 min') and "
 
-reverse=`psql -d nominatim -c "copy ($query type='reverse') to stdout"`
-searchy=`psql -d nominatim -c "copy ($query type='search' and results>0) to stdout"`
-searchn=`psql -d nominatim -c "copy ($query type='search' and results=0) to stdout"`
-details=`psql -d nominatim -c "copy ($query type='details') to stdout"`
+reverse=`psql -c "copy ($query type='reverse') to stdout"`
+searchy=`psql -c "copy ($query type='search' and results>0) to stdout"`
+searchn=`psql -c "copy ($query type='search' and results=0) to stdout"`
+details=`psql -c "copy ($query type='details') to stdout"`
  
 echo "z1.value $reverse"
 echo "z2.value $searchy"

mysql2pgsql/mysql2pgsql.perl

@@ -0,0 +1,949 @@
+#!/usr/bin/perl -w
+# mysql2pgsql
+# MySQL to PostgreSQL dump file converter
+#
+# For usage: perl mysql2pgsql.perl --help
+#
+# ddl statments are changed but none or only minimal real data
+# formatting are done.
+# data consistency is up to the DBA.
+#
+# (c) 2004-2007 Jose M Duarte and Joseph Speigle ... gborg
+#
+# (c) 2000-2004 Maxim Rudensky  <fonin@omnistaronline.com>
+# (c) 2000 Valentine Danilchuk  <valdan@ziet.zhitomir.ua>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. All advertising materials mentioning features or use of this software
+#    must display the following acknowledgement:
+# This product includes software developed by the Max Rudensky
+# and its contributors.
+# 4. Neither the name of the author nor the names of its contributors
+#    may be used to endorse or promote products derived from this software
+#    without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+use Getopt::Long;
+
+use POSIX;
+
+use strict;
+use warnings;
+
+
+# main sections
+# -------------
+# 1 variable declarations
+# 2 subroutines
+# 3 get commandline options and specify help statement
+# 4 loop through file and process
+# 5. print_plpgsql function prototype
+
+#################################################################
+#  1.  variable declarations
+#################################################################
+# command line options
+my( $ENC_IN, $ENC_OUT, $PRESERVE_CASE, $HELP, $DEBUG, $SCHEMA, $LOWERCASE, $CHAR2VARCHAR, $NODROP, $SEP_FILE, $opt_debug, $opt_help, $opt_schema, $opt_preserve_case, $opt_char2varchar, $opt_nodrop, $opt_sepfile, $opt_enc_in, $opt_enc_out );
+# variables for constructing pre-create-table entities
+my $pre_create_sql='';    # comments, 'enum' constraints preceding create table statement
+my $auto_increment_seq= '';    # so we can easily substitute it if we need a default value
+my $create_sql='';    # all the datatypes in the create table section
+my $post_create_sql='';   # create indexes, foreign keys, table comments
+my $function_create_sql = '';  # for the set (function,trigger) and CURRENT_TIMESTAMP ( function,trigger )
+#  constraints
+my ($type, $column_valuesStr, @column_values, $value );
+my %constraints=(); #  holds values constraints used to emulate mysql datatypes (e.g. year, set)
+# datatype conversion variables
+my ( $index,$seq);
+my ( $column_name, $col, $quoted_column);
+my ( @year_holder, $year, $constraint_table_name);
+my $table="";   # table_name for create sql statements
+my $table_no_quotes="";   # table_name for create sql statements
+my $sl = '^\s+\w+\s+';  # matches the column name
+my $tables_first_timestamp_column= 1;  #  decision to print warnings about default_timestamp not being in postgres
+my $mysql_numeric_datatypes = "TINYINT|SMALLINT|MEDIUMINT|INT|INTEGER|BIGINT|REAL|DOUBLE|FLOAT|DECIMAL|NUMERIC";
+my $mysql_datetime_datatypes = "|DATE|TIME|TIMESTAMP|DATETIME|YEAR";
+my $mysql_text_datatypes = "CHAR|VARCHAR|BINARY|VARBINARY|TINYBLOB|BLOB|MEDIUMBLOB|LONGBLOB|TINYTEXT|TEXT|MEDIUMTEXT|LONGTEXT|ENUM|SET";
+my $mysql_datatypesStr =  $mysql_numeric_datatypes . "|". $mysql_datetime_datatypes . "|". $mysql_text_datatypes ;
+# handling INSERT INTO statements
+my $rowRe = qr{
+    \(                  # opening parens
+        (               #  (start capture)
+            (?:         #  (start group)
+            '           # string start
+                [^'\\]*     # up to string-end or backslash (escape)
+                (?:     #  (start group)
+                \\.     # gobble escaped character
+                [^'\\]*     # up to string-end of backslash
+                )*      #  (end group, repeat zero or more)
+            '           # string end
+            |           #  (OR)
+            .*?         # everything else (not strings)
+            )*          #  (end group, repeat zero or more)
+        )               #  (end capture)
+    \)                  # closing parent
+}x;
+
+my ($insert_table, $valueString);
+#
+########################################################
+# 2.  subroutines
+#
+# get_identifier
+# print_post_create_sql()
+# quote_and_lc()
+# make_plpgsql($table,$column_name) -- at end of file
+########################################################
+
+# returns an identifier with the given suffix doing controlled
+# truncation if necessary
+sub get_identifier($$$) {
+    my ($table, $col, $suffix) = @_;
+    my $name = '';
+    $table=~s/\"//g; # make sure that $table doesn't have quotes so we don't end up with redundant quoting
+    # in the case of multiple columns
+    my @cols = split(/,/,$col);
+    $col =~ s/,//g;
+    # in case all columns together too long we have to truncate them
+    if (length($col) > 55) {
+        my $totaltocut = length($col)-55;
+        my $tocut = ceil($totaltocut / @cols);
+        @cols = map {substr($_,0,abs(length($_)-$tocut))} @cols;
+        $col="";
+        foreach (@cols){
+            $col.=$_;
+        }
+    }
+
+    my $max_table_length = 63 - length("_${col}_$suffix");
+
+    if (length($table) > $max_table_length) {
+        $table = substr($table, length($table) - $max_table_length, $max_table_length);
+    }
+    return quote_and_lc("${table}_${col}_${suffix}");
+}
+
+
+#
+#
+# called when we encounter next CREATE TABLE statement
+# also called at EOF to print out for last table
+# prints comments, indexes, foreign key constraints (the latter 2 possibly to a separate file)
+sub print_post_create_sql() {
+    my ( @create_idx_comments_constraints_commandsArr, $stmts, $table_field_combination);
+    my %stmts;
+    # loop to check for duplicates in $post_create_sql
+    # Needed because of duplicate key declarations ( PRIMARY KEY and KEY), auto_increment columns
+
+    @create_idx_comments_constraints_commandsArr = split(';\n?', $post_create_sql);
+    if ($SEP_FILE) {
+        open(SEP_FILE, ">>:encoding($ENC_OUT)", $SEP_FILE) or die "Unable to open $SEP_FILE for output: $!\n";
+    }
+
+    foreach (@create_idx_comments_constraints_commandsArr) {
+        if (m/CREATE INDEX "*(\S+)"*\s/i) {  #  CREATE INDEX korean_english_wordsize_idx ON korean_english USING btree  (wordsize);
+            $table_field_combination =  $1;
+            # if this particular table_field_combination was already used do not print the statement:
+            if ($SEP_FILE) {
+                print SEP_FILE "$_;\n" if !defined($stmts{$table_field_combination});
+            } else {
+                print OUT "$_;\n" if !defined($stmts{$table_field_combination});
+            }
+            $stmts{$table_field_combination} = 1;
+        }
+        elsif (m/COMMENT/i) {  # COMMENT ON object IS 'text'; but comment may be part of table name so use 'elsif'
+            print OUT "$_;\n"
+        } else {  # foreign key constraint  or comments (those preceded by -- )
+            if ($SEP_FILE) {
+                print SEP_FILE "$_;\n";
+            } else {
+                print OUT "$_;\n"
+            }
+        }
+    }
+
+    if ($SEP_FILE) {
+        close SEP_FILE;
+    }
+    $post_create_sql='';
+    # empty %constraints for next " create table" statement
+}
+
+# quotes a string or a multicolumn string (comma separated)
+# and optionally lowercase (if LOWERCASE is set)
+# lowercase .... if user wants default postgres behavior
+# quotes .... to preserve keywords and to preserve case when case-sensitive tables are to be used
+sub quote_and_lc($)
+{
+    my $col = shift;
+    if ($LOWERCASE) {
+        $col = lc($col);
+    }
+    if ($col =~ m/,/) {
+        my @cols = split(/,\s?/, $col);
+        @cols = map {"\"$_\""} @cols;
+        return join(', ', @cols);
+    } else {
+        return "\"$col\"";
+    }
+}
+
+########################################################
+# 3.  get commandline options and maybe print help
+########################################################
+
+GetOptions("help", "debug"=> \$opt_debug, "schema=s" => \$SCHEMA, "preserve_case" => \$opt_preserve_case, "char2varchar" => \$opt_char2varchar, "nodrop" => \$opt_nodrop, "sepfile=s" => \$opt_sepfile, "enc_in=s" => \$opt_enc_in, "enc_out=s" => \$opt_enc_out );
+
+$HELP = $opt_help || 0;
+$DEBUG = $opt_debug || 0;
+$PRESERVE_CASE = $opt_preserve_case || 0;
+if ($PRESERVE_CASE == 1) { $LOWERCASE = 0; }
+else { $LOWERCASE = 1; }
+$CHAR2VARCHAR = $opt_char2varchar || 0;
+$NODROP = $opt_nodrop || 0;
+$SEP_FILE = $opt_sepfile || 0;
+$ENC_IN = $opt_enc_in || 'utf8';
+$ENC_OUT = $opt_enc_out || 'utf8';
+
+if (($HELP) || ! defined($ARGV[0]) || ! defined($ARGV[1])) {
+    print "\n\nUsage: perl $0 {--help --debug --preserve_case --char2varchar --nodrop --schema --sepfile --enc_in --enc_out } mysql.sql pg.sql\n";
+    print "\t* OPTIONS WITHOUT ARGS\n";
+    print "\t--help:  prints this message \n";
+    print "\t--debug: output the commented-out mysql line above the postgres line in pg.sql \n";
+    print "\t--preserve_case: prevents automatic case-lowering of column and table names\n";
+    print "\t\tIf you want to preserve case, you must set this flag. For example,\n";
+    print "\t\tIf your client application quotes table and column-names and they have cases in them, set this flag\n";
+    print "\t--char2varchar: converts all char fields to varchar\n";
+    print "\t--nodrop: strips out DROP TABLE statements\n";
+    print "\t\totherise harmless warnings are printed by psql when the dropped table does not exist\n";
+    print "\n\t* OPTIONS WITH ARGS\n";
+    print "\t--schema: outputs a line into the postgres sql file setting search_path \n";
+    print "\t--sepfile: output foreign key constraints and indexes to a separate file so that it can be\n";
+    print "\t\timported after large data set is inserted from another dump file\n";
+    print "\t--enc_in: encoding of mysql in file (default utf8) \n";
+    print "\t--enc_out: encoding of postgres out file (default utf8) \n";
+    print "\n\t* REQUIRED ARGUMENTS\n";
+    if (defined ($ARGV[0])) {
+        print "\tmysql.sql ($ARGV[0])\n";
+    } else {
+        print "\tmysql.sql (undefined)\n";
+    }
+    if (defined ($ARGV[1])) {
+        print "\tpg.sql ($ARGV[1])\n";
+    } else {
+        print "\tpg.sql (undefined)\n";
+    }
+    print "\n";
+    exit 1;
+}
+########################################################
+# 4.  process through mysql_dump.sql file
+# in a big loop
+########################################################
+
+# open in and out files
+open(IN,"<:encoding($ENC_IN)", $ARGV[0]) || die "can't open mysql dump file $ARGV[0]";
+open(OUT,">:encoding($ENC_OUT)", $ARGV[1]) || die "can't open pg dump file $ARGV[1]";
+
+# output header
+print OUT "--\n";
+print OUT "-- Generated from mysql2pgsql.perl\n";
+print OUT "-- http://gborg.postgresql.org/project/mysql2psql/\n";
+print OUT "-- (c) 2001 - 2007 Jose M. Duarte, Joseph Speigle\n";
+print OUT "--\n";
+print OUT "\n";
+print OUT "-- warnings are printed for drop tables if they do not exist\n";
+print OUT "-- please see http://archives.postgresql.org/pgsql-novice/2004-10/msg00158.php\n\n";
+print OUT "-- ##############################################################\n";
+
+if ($SCHEMA ) {
+    print OUT "set search_path='" . $SCHEMA . "'\\g\n" ;
+}
+
+# loop through mysql file  on a per-line basis
+while(<IN>) {
+
+##############     flow     #########################
+# (the lines are directed to different string variables at different times)
+#
+# handle drop table , unlock, connect statements
+# if ( start of create table)   {
+#   print out post_create table (indexes, foreign key constraints, comments from previous table)
+#   add drop table statement if !$NODROP to pre_create_sql
+#   next;
+# }
+# else if ( inside create table) {
+#   add comments in this portion to create_sql
+#   if ( end of create table) {
+#      delete mysql-unique CREATE TABLE commands
+#      print pre_create_sql
+#      print the constraint tables for set and year datatypes
+#      print create_sql
+#      print function_create_sql (this is for the enum columns only)
+#      next;
+#   }
+#   do substitutions
+#    -- NUMERIC DATATYPES
+#    -- CHARACTER DATATYPES
+#    -- DATE AND TIME DATATYPES
+#    -- KEY AND UNIQUE CREATIONS
+#    and append them to create_sql
+# } else {
+#   print inserts on-the-spot (this script only changes default timestamp of 0000-00-00)
+# }
+# LOOP until EOF
+#
+########################################################
+
+
+if (!/^\s*insert into/i) { # not inside create table so don't worry about data corruption
+    s/`//g;  #  '`pgsql uses no backticks to denote table name (CREATE TABLE `sd`) or around field
+            # and table names like  mysql
+            # doh!  we hope all dashes and special chars are caught by the regular expressions :)
+}
+if (/^\s*USE\s*([^;]*);/) {
+    print OUT "\\c ". $1;
+    next;
+}
+if (/^(UN)?LOCK TABLES/i  || /drop\s+table/i ) {
+
+    # skip
+    # DROP TABLE is added when we see the CREATE TABLE
+    next;
+}
+if (/(create\s+table\s+)([-_\w]+)\s/i) { #  example: CREATE TABLE `english_english`
+    print_post_create_sql();   # for last table
+    $tables_first_timestamp_column= 1;  #  decision to print warnings about default_timestamp not being in postgres
+    $create_sql = '';
+    $table_no_quotes = $2 ;
+    $table=quote_and_lc($2);
+    if ( !$NODROP )  {  # always print drop table if user doesn't explicitly say not to
+        #  to drop a table that is referenced by a view or a foreign-key constraint of another table,
+        #  CASCADE must be specified. (CASCADE will remove a dependent view entirely, but in the
+        #  in the foreign-key case it will only remove the foreign-key constraint, not the other table entirely.)
+        #  (source: 8.1.3 docs, section "drop table")
+        warn "table $table will be dropped CASCADE\n";
+        $pre_create_sql .= "DROP TABLE $table CASCADE\\g\n";    # custom dumps may be missing the 'dump' commands
+    }
+
+    s/(create\s+table\s+)([-_\w]+)\s/$1 $table /i;
+    if ($DEBUG) {
+        $create_sql .=  '-- ' . $_;
+    }
+    $create_sql .= $_;
+    next;
+}
+if ($create_sql ne "") {         # we are inside create table statement so lets process datatypes
+    # print out comments or empty lines in context
+    if ($DEBUG) {
+        $create_sql .=  '-- ' . $_;
+    }
+    if (/^#/ || /^$/ || /^\s*--/) {
+        s/^#/--/;   #  Two hyphens (--) is the SQL-92 standard indicator for comments
+        $create_sql.=$_;
+        next;
+    }
+
+    if (/\).*;/i) {    # end of create table squence
+
+        s/INSERT METHOD[=\s+][^;\s]+//i;
+        s/PASSWORD=[^;\s]+//i;
+        s/ROW_FORMAT=(?:DEFAULT|DYNAMIC|FIXED|COMPRESSED|REDUNDANT|COMPACT)+//i;
+        s/DELAY KEY WRITE=[^;\s]+//i;
+        s/INDEX DIRECTORY[=\s+][^;\s]+//i;
+        s/DATA DIRECTORY=[^;\s]+//i;
+        s/CONNECTION=[^;\s]+//i;
+        s/CHECKSUM=[^;\s]+//i;
+        s/Type=[^;\s]+//i; # ISAM ,   # older versions
+        s/COLLATE=[^;\s]+//i;         # table's collate
+        s/COLLATE\s+[^;\s]+//i;         # table's collate
+        # possible AUTO_INCREMENT starting index, it is used in mysql 5.0.26, not sure since which version
+        if (/AUTO_INCREMENT=(\d+)/i) {
+        # should take < ----  ) ENGINE=MyISAM AUTO_INCREMENT=16 DEFAULT CHARSET=latin1;
+        # and should ouput --->  CREATE SEQUENCE "rhm_host_info_id_seq" START WITH 16;
+        my $start_value = $1;
+        print $auto_increment_seq . "--\n";
+        # print $pre_create_sql . "--\n";
+        $pre_create_sql =~ s/(CREATE SEQUENCE $auto_increment_seq )/$1 START WITH $start_value /;
+    }
+        s/AUTO_INCREMENT=\d+//i;
+        s/PACK_KEYS=\d//i;            # mysql 5.0.22
+        s/DEFAULT CHARSET=[^;\s]+//i; #  my mysql version is 4.1.11
+        s/ENGINE\s*=\s*[^;\s]+//i;   #  my mysql version is 4.1.11
+        s/ROW_FORMAT=[^;\s]+//i;   #  my mysql version is 5.0.22
+        s/MIN_ROWS=[^;\s]+//i;
+        s/MAX_ROWS=[^;\s]+//i;
+        s/AVG_ROW_LENGTH=[^;\s]+//i;
+        if (/COMMENT='([^']*)'/) {  # ) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='must be country zones';
+            $post_create_sql.="COMMENT ON TABLE $table IS '$1'\;"; # COMMENT ON table_name IS 'text';
+            s/COMMENT='[^']*'//i;
+        }
+        $create_sql =~ s/,$//g;    # strip last , inside create table
+        # make sure we end in a comma, as KEY statments are turned
+        # into post_create_sql indices
+        # they often are the last line so leaving a 'hanging comma'
+        my @array = split("\n", $create_sql);
+        for (my $a = $#array; $a >= 0; $a--) {  #loop backwards
+            if ($a == $#array  && $array[$a] =~ m/,\s*$/) {    # for last line
+                $array[$a] =~ s/,\s*$//;
+                next;
+            }
+            if ($array[$a] !~ m/create table/i) {  # i.e. if there was more than one column in table
+                if ($a != $#array  && $array[$a] !~ m/,\s*$/  ) {  # for second to last
+                    $array[$a] =~ s/$/,/;
+                    last;
+                }
+                elsif ($a != $#array  && $array[$a] =~ m/,\s*$/ ) {  # for second to last
+                    last;
+                }
+            }
+        }
+        $create_sql = join("\n", @array) . "\n";
+        $create_sql .=  $_;
+
+        # put comments out first
+        print OUT $pre_create_sql;
+
+        # create separate table to reference and to hold mysql's possible set data-type
+        # values.  do that table's creation before create table
+        # definition
+        foreach $column_name (keys %constraints) {
+            $type=$constraints{$column_name}{'type'};
+            $column_valuesStr = $constraints{$column_name}{'values'};
+            $constraint_table_name = get_identifier(${table},${column_name} ,"constraint_table");
+            if ($type eq 'set') {
+                print OUT qq~DROP TABLE $constraint_table_name  CASCADE\\g\n~ ;
+                print OUT qq~create table $constraint_table_name  ( set_values varchar UNIQUE)\\g\n~ ;
+                $function_create_sql .= make_plpgsql($table,$column_name);
+            } elsif ($type eq 'year')  {
+                print OUT qq~DROP TABLE $constraint_table_name  CASCADE\\g\n~ ;
+                print OUT qq~create table $constraint_table_name  ( year_values varchar UNIQUE)\\g\n~ ;
+            }
+            @column_values = split /,/, $column_valuesStr;
+            foreach $value (@column_values) {
+                print OUT qq~insert into $constraint_table_name   values (  $value  )\\g\n~; # ad ' for ints and varchars
+            }
+        }
+
+        $create_sql =~ s/double double/double precision/g;
+
+        # print create table and reset create table vars
+        # when moving from each "create table" to "insert" part of dump
+        print OUT $create_sql;
+        print OUT $function_create_sql;
+        $pre_create_sql="";
+        $auto_increment_seq="";
+        $create_sql="";
+        $function_create_sql='';
+        %constraints=();
+        # the post_create_sql for this table is output at the beginning of the next table def
+        # in case we want to make indexes after doing inserting
+        next;
+    }
+    if (/^\s*(\w+)\s+.*COMMENT\s*'([^']*)'/) {  #`zone_country_id` int(11) COMMENT 'column comment here',
+        $quoted_column=quote_and_lc($1);
+        $post_create_sql.="COMMENT ON COLUMN $table"."."." $quoted_column IS '$2'\;"; # COMMENT ON table_name.column_name IS 'text';
+        s/COMMENT\s*'[^']*'//i;
+    }
+
+
+    # NUMERIC DATATYPES
+    #
+    # auto_increment -> sequences
+    # UNSIGNED conversions
+    # TINYINT
+    # SMALLINT
+    # MEDIUMINT
+    # INT, INTEGER
+    # BIGINT
+    #
+    # DOUBLE [PRECISION], REAL
+    # DECIMAL(M,D), NUMERIC(M,D)
+    # FLOAT(p)
+    # FLOAT
+
+    s/(\w*int)\(\d+\)/$1/g;  # hack of the (n) stuff for e.g. mediumint(2) int(3)
+
+    if (/^(\s*)(\w+)\s*.*numeric.*auto_increment/i) {         # int,auto_increment -> serial
+        $seq = get_identifier($table, $2, 'seq');
+        $quoted_column=quote_and_lc($2);
+        # Smash datatype to int8 and autogenerate the sequence.
+        s/^(\s*)(\w+)\s*.*NUMERIC(.*)auto_increment([^,]*)/$1 $quoted_column serial8 $4/ig;
+        $create_sql.=$_;
+        next;
+    }
+    if (/^\s*(\w+)\s+.*int.*auto_increment/i) {  #  example: data_id mediumint(8) unsigned NOT NULL auto_increment,
+        $seq = get_identifier($table, $1, 'seq');
+        $quoted_column=quote_and_lc($1);
+        s/(\s*)(\w+)\s+.*int.*auto_increment([^,]*)/$1 $quoted_column serial8 $3/ig;
+        $create_sql.=$_;
+        next;
+    }
+
+
+
+
+    # convert UNSIGNED to CHECK constraints
+    if (m/^(\s*)(\w+)\s+((float|double precision|double|real|decimal|numeric))(.*)unsigned/i) {
+        $quoted_column = quote_and_lc($2);
+        s/^(\s*)(\w+)\s+((float|double precision|double|real|decimal|numeric))(.*)unsigned/$1 $quoted_column $3 $4 CHECK ($quoted_column >= 0)/i;
+    }
+    # example:  `wordsize` tinyint(3) unsigned default NULL,
+    if (m/^(\s+)(\w+)\s+(\w+)\s+unsigned/i) {
+        $quoted_column=quote_and_lc($2);
+        s/^(\s+)(\w+)\s+(\w+)\s+unsigned/$1 $quoted_column $3 CHECK ($quoted_column >= 0)/i;
+    }
+    if (m/^(\s*)(\w+)\s+(bigint.*)unsigned/) {
+        $quoted_column=quote_and_lc($2);
+        #  see http://archives.postgresql.org/pgsql-general/2005-07/msg01178.php
+        #  and see http://www.postgresql.org/docs/8.2/interactive/datatype-numeric.html
+        # see  http://dev.mysql.com/doc/refman/5.1/en/numeric-types.html  max size == 20 digits
+        s/^(\s*)(\w+)\s+bigint(.*)unsigned/$1 $quoted_column NUMERIC (20,0) CHECK ($quoted_column >= 0)/i;
+
+    }
+
+    # int type conversion
+    # TINYINT    (signed) -128 to 127 (unsigned) 0   255
+    #  SMALLINT A small integer. The signed range is -32768 to 32767. The unsigned range is 0 to 65535.
+    #  MEDIUMINT  A medium-sized integer. The signed range is -8388608 to 8388607. The unsigned range is 0 to 16777215.
+    #  INT A normal-size integer. The signed range is -2147483648 to 2147483647. The unsigned range is 0 to 4294967295.
+    # BIGINT The signed range is -9223372036854775808 to 9223372036854775807. The unsigned range is 0 to 18446744073709551615
+    # for postgres see http://www.postgresql.org/docs/8.2/static/datatype-numeric.html#DATATYPE-INT
+    s/^(\s+"*\w+"*\s+)tinyint/$1 smallint/i;
+    s/^(\s+"*\w+"*\s+)mediumint/$1 integer/i;
+
+    # the floating point types
+    #   double -> double precision
+    #   double(n,m) -> double precision
+    #   float - no need for conversion
+    #   float(n) - no need for conversion
+    #   float(n,m) -> double precision
+
+    s/(^\s*\w+\s+)double(\(\d+,\d+\))?/$1float/i;
+    s/float(\(\d+,\d+\))/float/i;
+
+    #
+    # CHARACTER TYPES
+    #
+    # set
+    # enum
+    # binary(M), VARBINARy(M), tinyblob, tinytext,
+    # bit
+    # char(M), varchar(M)
+    # blob -> text
+    # mediumblob
+    # longblob, longtext
+    # text -> text
+    # mediumtext
+    # longtext
+    #  mysql docs: A BLOB is a binary large object that can hold a variable amount of data.
+
+    # set
+    # For example, a column specified as SET('one', 'two') NOT NULL can have any of these values:
+    # ''
+    # 'one'
+    # 'two'
+    # 'one,two'
+    if (/(\w*)\s+set\(((?:['"]\w+['"]\s*,*)+(?:['"]\w+['"])*)\)(.*)$/i) { # example:  `au_auth` set('r','w','d') NOT NULL default '',
+        $column_name = $1;
+        $constraints{$column_name}{'values'} = $2;  # 'abc','def', ...
+        $constraints{$column_name}{'type'} = "set";  # 'abc','def', ...
+        $_ =  qq~ $column_name varchar , ~;
+        $column_name = quote_and_lc($1);
+        $create_sql.=$_;
+        next;
+
+    }
+    if (/(\S*)\s+enum\(((?:['"][^'"]+['"]\s*,)+['"][^'"]+['"])\)(.*)$/i) { # enum handling
+        #  example:  `test` enum('?','+','-') NOT NULL default '?'
+        # $2  is the values of the enum 'abc','def', ...
+        $quoted_column=quote_and_lc($1);
+        #  "test" NOT NULL default '?' CONSTRAINT test_test_constraint CHECK ("test" IN ('?','+','-'))
+        $_ = qq~ $quoted_column varchar CHECK ($quoted_column IN ( $2 ))$3\n~;  # just assume varchar?
+        $create_sql.=$_;
+        next;
+    }
+    # Take care of "binary" option for char and varchar
+    # (pre-4.1.2, it indicated a byte array; from 4.1.2, indicates
+    # a binary collation)
+    s/(?:var)?char(?:\(\d+\))? (?:byte|binary)/text/i;
+    if (m/(?:var)?binary\s*\(\d+\)/i) {   #  c varBINARY(3) in Mysql
+        warn "WARNING in table '$table' '$_':  binary type is converted to bytea (unsized) for Postgres\n";
+    }
+    s/(?:var)?binary(?:\(\d+\))?/text/i;   #  c varBINARY(3) in Mysql
+    s/bit(?:\(\d+\))?/bytea/i;   #  bit datatype -> bytea
+
+    # large datatypes
+    s/\w*blob/bytea/gi;
+    s/tinytext/text/gi;
+    s/mediumtext/text/gi;
+    s/longtext/text/gi;
+
+    # char -> varchar -- if specified as a command line option
+    # PostgreSQL would otherwise pad with spaces as opposed
+    # to MySQL! Your user interface may depend on this!
+    if ($CHAR2VARCHAR) {
+        s/(^\s+\S+\s+)char/${1}varchar/gi;
+    }
+
+    # nuke column's collate and character set
+    s/(\S+)\s+character\s+set\s+\w+/$1/gi;
+    s/(\S+)\s+collate\s+\w+/$1/gi;
+
+    #
+    # DATE AND TIME TYPES
+    #
+    # date  time
+    # year
+    # datetime
+    # timestamp
+
+    # date  time
+    # these are the same types in postgres, just do the replacement of 0000-00-00 date
+
+    if (m/default '(\d+)-(\d+)-(\d+)([^']*)'/i) { # we grab the year, month and day
+        # NOTE: times of 00:00:00 are possible and are okay
+        my $time = '';
+        my $year=$1;
+        my $month= $2;
+        my $day = $3;
+        if ($4) {
+            $time = $4;
+        }
+        if ($year eq "0000") { $year = '1970'; }
+        if ($month eq "00") { $month = '01'; }
+        if ($day eq "00") { $day = '01'; }
+        s/default '[^']+'/default '$year-$month-$day$time'/i; # finally we replace with $datetime
+    }
+
+    # convert mysql's year datatype to a constraint
+    if (/(\w*)\s+year\(4\)(.*)$/i) { # can be integer OR string 1901-2155
+        $constraint_table_name = get_identifier($table,$1 ,"constraint_table");
+        $column_name=quote_and_lc($1);
+        @year_holder = ();
+        $year='';
+        for (1901 .. 2155) {
+                $year = "'$_'";
+            unless ($year =~ /2155/) { $year .= ','; }
+             push( @year_holder, $year);
+        }
+        $constraints{$column_name}{'values'} = join('','',@year_holder);   # '1901','1902', ...
+        $constraints{$column_name}{'type'} = "year";
+        $_ =  qq~ $column_name varchar CONSTRAINT ${table}_${column_name}_constraint REFERENCES $constraint_table_name ("year_values") $2\n~;
+        $create_sql.=$_;
+        next;
+    } elsif (/(\w*)\s+year\(2\)(.*)$/i) { # same for a 2-integer string
+        $constraint_table_name = get_identifier($table,$1 ,"constraint_table");
+        $column_name=quote_and_lc($1);
+        @year_holder = ();
+        $year='';
+        for (1970 .. 2069) {
+            $year = "'$_'";
+            if ($year =~ /2069/) { next; }
+            push( @year_holder, $year);
+        }
+        push( @year_holder, '0000');
+        $constraints{$column_name}{'values'} = join(',',@year_holder);   # '1971','1972', ...
+        $constraints{$column_name}{'type'} = "year";  # 'abc','def', ...
+        $_ =  qq~ $1 varchar CONSTRAINT ${table}_${column_name}_constraint REFERENCES $constraint_table_name ("year_values") $2\n~;
+        $create_sql.=$_;
+        next;
+    }
+
+    # datetime
+    # Default on a dump from MySQL 5.0.22 is in the same form as datetime so let it flow down
+    # to the timestamp section and deal with it there
+    s/(${sl})datetime /$1timestamp without time zone /i;
+
+    # change not null datetime field to null valid ones
+    # (to support remapping of "zero time" to null
+    # s/($sl)datetime not null/$1timestamp without time zone/i;
+
+
+    # timestamps
+    #
+    # nuke datetime representation (not supported in PostgreSQL)
+    # change default time of 0000-00-00 to 1970-01-01
+
+    # we may possibly need to create a trigger to provide
+    # equal functionality with ON UPDATE CURRENT TIMESTAMP
+
+
+    if (m/${sl}timestamp/i) {
+        if ( m/ON UPDATE CURRENT_TIMESTAMP/i )  {  # the ... default CURRENT_TIMESTAMP  only applies for blank inserts, not updates
+            s/ON UPDATE CURRENT_TIMESTAMP//i ;
+            m/^\s*(\w+)\s+timestamp/i ;
+            # automatic trigger creation
+            $table_no_quotes =~ s/"//g;
+$function_create_sql .= " CREATE OR REPLACE FUNCTION update_". $table_no_quotes . "() RETURNS trigger AS '
+BEGIN
+    NEW.$1 := CURRENT_TIMESTAMP;
+    RETURN NEW;
+END;
+' LANGUAGE 'plpgsql';
+
+-- before INSERT is handled by 'default CURRENT_TIMESTAMP'
+CREATE TRIGGER add_current_date_to_".$table_no_quotes." BEFORE UPDATE ON ". $table . " FOR EACH ROW EXECUTE PROCEDURE
+update_".$table_no_quotes."();\n";
+
+        }
+        if ($tables_first_timestamp_column && m/DEFAULT NULL/i) {
+            # DEFAULT NULL is the same as DEFAULT CURRENT_TIMESTAMP for the first TIMESTAMP  column. (MYSQL manual)
+            s/($sl)(timestamp\s+)default null/$1 $2 DEFAULT CURRENT_TIMESTAMP/i;
+        }
+        $tables_first_timestamp_column= 0;
+        if (m/${sl}timestamp\s*\(\d+\)/i) {   # fix for timestamps with width spec not handled (ID: 1628)
+            warn "WARNING for in table '$table' '$_': your default timestamp width is being ignored for table $table \n";
+            s/($sl)timestamp(?:\(\d+\))/$1datetime/i;
+        }
+    } # end timestamp section
+
+    # KEY AND UNIQUE CREATIONS
+    #
+    # unique
+    if ( /^\s+unique\s+\(([^(]+)\)/i ) { #  example    UNIQUE `name` (`name`), same as UNIQUE KEY
+        #  POSTGRESQL:  treat same as mysql unique
+        $quoted_column = quote_and_lc($1);
+        s/\s+unique\s+\(([^(]+)\)/ unique ($quoted_column) /i;
+            $create_sql.=$_;
+        next;
+        } elsif ( /^\s+unique\s+key\s*(\w+)\s*\(([^(]+)\)/i ) { #  example    UNIQUE KEY `name` (`name`)
+            #  MYSQL: unique  key: allows null=YES, allows duplicates=NO (*)
+            #  ... new ... UNIQUE KEY `unique_fullname` (`fullname`)  in my mysql v. Ver 14.12 Distrib 5.1.7-beta
+            #  POSTGRESQL:  treat same as mysql unique
+        # just quote columns
+        $quoted_column = quote_and_lc($2);
+            s/\s+unique\s+key\s*(\w+)\s*\(([^(]+)\)/ unique ($quoted_column) /i;
+            $create_sql.=$_;
+        # the index corresponding to the 'key' is automatically created
+            next;
+    }
+    # keys
+    if ( /^\s+fulltext key\s+/i) { # example:  FULLTEXT KEY `commenttext` (`commenttext`)
+    # that is key as a word in the first check for a match
+        # the tsvector datatype is made for these types of things
+        # example mysql file:
+        #  what is tsvector datatype?
+        #  http://www.sai.msu.su/~megera/postgres/gist/tsearch/V2/docs/tsearch-V2-intro.html
+        warn "dba must do fulltext key transformation for $table\n";
+        next;
+    }
+    if ( /^(\s+)constraint (\S+) foreign key \((\S+)\) references (\S+) \((\S+)\)(.*)/i ) {
+        $quoted_column =quote_and_lc($3);
+        $col=quote_and_lc($5);
+        $post_create_sql .= "ALTER TABLE $table ADD FOREIGN KEY ($quoted_column) REFERENCES " . quote_and_lc($4) . " ($col);\n";
+        next;
+    }
+    if ( /^\s*primary key\s*\(([^)]+)\)([,\s]+)/i ) { #  example    PRIMARY KEY (`name`)
+        # MYSQL: primary key: allows null=NO , allows duplicates=NO
+        #  POSTGRESQL: When an index is declared unique, multiple table rows with equal indexed values will not be
+        #       allowed. Null values are not considered equal.
+        #  POSTGRESQL quote's source: 8.1.3 docs section 11.5 "unique indexes"
+        #  so, in postgres, we need to add a NOT NULL to the UNIQUE constraint
+        # and, primary key (mysql) == primary key (postgres) so that we *really* don't need change anything
+        $quoted_column = quote_and_lc($1);
+        s/(\s*)primary key\s+\(([^)]+)\)([,\s]+)/$1 primary key ($quoted_column)$3/i;
+        # indexes are automatically created for unique columns
+        $create_sql.=$_;
+        next;
+    } elsif (m/^\s+key\s[-_\s\w]+\((.+)\)/i    ) {     # example:   KEY `idx_mod_english_def_word` (`word`),
+        # regular key: allows null=YES, allows duplicates=YES
+        # MYSQL:   KEY is normally a synonym for INDEX.  http://dev.mysql.com/doc/refman/5.1/en/create-table.html
+        #
+        #  * MySQL: ALTER TABLE {$table} ADD KEY $column ($column)
+        #  * PostgreSQL: CREATE INDEX {$table}_$column_idx ON {$table}($column) // Please note the _idx "extension"
+        #    PRIMARY KEY (`postid`),
+        #    KEY `ownerid` (`ownerid`)
+        # create an index for everything which has a key listed for it.
+        my $col = $1;
+        # TODO we don't have a translation for the substring syntax in text columns in MySQL (e.g. "KEY my_idx (mytextcol(20))")
+        # for now just getting rid of the brackets and numbers (the substring specifier):
+        $col=~s/\(\d+\)//g;
+        $quoted_column = quote_and_lc($col);
+        if ($col =~ m/,/) {
+            $col =  s/,/_/;
+        }
+        $index = get_identifier($table, $col, 'idx');
+        $post_create_sql.="CREATE INDEX $index ON $table USING btree ($quoted_column)\;";
+        # just create index do not add to create table statement
+        next;
+    }
+
+    # handle 'key' declared at end of column
+    if (/\w+.*primary key/i) {   # mysql: key is normally just a synonym for index
+    # just leave as is ( postgres has primary key type)
+
+
+    } elsif (/(\w+\s+(?:$mysql_datatypesStr)\s+.*)key/i) {   # mysql: key is normally just a synonym for index
+    # I can't find a reference for 'key' in a postgres command without using the word 'primary key'
+        s/$1key/$1/i ;
+        $index = get_identifier($table, $1, 'idx');
+        $quoted_column =quote_and_lc($1);
+        $post_create_sql.="CREATE INDEX $index ON $table USING btree ($quoted_column) \;";
+        $create_sql.=$_;
+    }
+
+
+
+    # do we really need this anymore?
+    # remap colums with names of existing system attribute
+    if (/"oid"/i) {
+        s/"oid"/"_oid"/g;
+        print STDERR "WARNING: table $table uses column \"oid\" which is renamed to \"_oid\"\nYou should fix application manually! Press return to continue.";
+        my $wait=<STDIN>;
+    }
+
+    s/oid/_oid/i if (/key/i && /oid/i); # fix oid in key
+
+    # FINAL QUOTING OF ALL COLUMNS
+    # quote column names which were not already quoted
+    # perhaps they were not quoted because they were not explicitly handled
+    if (!/^\s*"(\w+)"(\s+)/i) {
+        /^(\s*)(\w+)(\s+)(.*)$/i ;
+        $quoted_column= quote_and_lc($2);
+        s/^(\s*)(\w+)(\s+)(.*)$/$1 $quoted_column $3 $4 /;
+    }
+    $create_sql.=$_;
+    #  END of if ($create_sql ne "") i.e. were inside create table statement so processed datatypes
+}
+# add "not in create table" comments or empty lines to pre_create_sql
+elsif (/^#/ || /^$/ || /^\s*--/) {
+    s/^#/--/;   #  Two hyphens (--) is the SQL-92 standard indicator for comments
+    $pre_create_sql .=  $_ ;  # printed above create table statement
+    next;
+}
+elsif (/^\s*insert into/i) { # not inside create table and doing insert
+    # fix mysql's zero/null value for timestamps
+    s/'0000-00-00/'1970-01-01/gi;
+    # commented out to fix bug "Field contents interpreted as a timestamp", what was the point of this line anyway?
+    #s/([12]\d\d\d)([01]\d)([0-3]\d)([0-2]\d)([0-6]\d)([0-6]\d)/'$1-$2-$3 $4:$5:$6'/;
+
+    #---- fix data in inserted data: (from MS world)
+    s!\x96!-!g;    # --
+    s!\x93!"!g;    # ``
+    s!\x94!"!g;    # ''
+    s!\x85!... !g;    # \ldots
+    s!\x92!`!g;
+
+    print OUT $pre_create_sql;    # print comments preceding the insert section
+    $pre_create_sql="";
+    $auto_increment_seq = "";
+
+    s/'((?:[^'\\]++|\\.)*+)'(?=[),])/E'$1'/g;
+    # for the E'' see http://www.postgresql.org/docs/8.2/interactive/release-8-1.html
+    s!\\\\!\\\\\\\\!g;      # replace \\ with ]\\\\
+
+    # split 'extended' INSERT INTO statements to something PostgreSQL can  understand
+    ( $insert_table,  $valueString) = $_ =~ m/^INSERT\s+INTO\s+['`"]*(.*?)['`"]*\s+VALUES\s*(.*)/i;
+    $insert_table = quote_and_lc($insert_table);
+
+    s/^INSERT INTO.*?\);//i;  # hose the statement which is to be replaced whether a run-on or not
+    # guarantee table names are quoted
+    print OUT qq(INSERT INTO $insert_table VALUES $valueString \n);
+
+} else {
+    print OUT $_ ;  #  example: /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+}
+#  keep looping and get next line of IN file
+
+} # END while(<IN>)
+
+print_post_create_sql();   # in case there is extra from the last table
+
+#################################################################
+#  5.  print_plgsql function prototype
+#      emulate the set datatype with the following plpgsql function
+#      looks ugly so putting at end of file
+#################################################################
+#
+sub make_plpgsql {
+my ($table,$column_name) = ($_[0],$_[1]);
+$table=~s/\"//g; # make sure that $table doesn't have quotes so we don't end up with redundant quoting
+my $constraint_table = get_identifier($table,$column_name ,"constraint_table");
+return "
+-- this function is called by the insert/update trigger
+-- it checks if the INSERT/UPDATE for the 'set' column
+-- contains members which comprise a valid mysql set
+-- this TRIGGER function therefore acts like a constraint
+--  provided limited functionality for mysql's set datatype
+-- just verifies and matches for string representations of the set at this point
+-- though the set datatype uses bit comparisons, the only supported arguments to our
+-- set datatype are VARCHAR arguments
+-- to add a member to the set add it to the ".$table."_".$column_name." table
+CREATE OR REPLACE FUNCTION check_".$table."_".$column_name."_set(  ) RETURNS TRIGGER AS \$\$\n
+DECLARE
+----
+arg_str VARCHAR ;
+argx VARCHAR := '';
+nobreak INT := 1;
+rec_count INT := 0;
+psn INT := 0;
+str_in VARCHAR := NEW.$column_name;
+----
+BEGIN
+----
+IF str_in IS NULL THEN RETURN NEW ; END IF;
+arg_str := REGEXP_REPLACE(str_in, '\\',\\'', ',');  -- str_in is CONSTANT
+arg_str := REGEXP_REPLACE(arg_str, '^\\'', '');
+arg_str := REGEXP_REPLACE(arg_str, '\\'\$', '');
+-- RAISE NOTICE 'arg_str %',arg_str;
+psn := POSITION(',' in arg_str);
+IF psn > 0 THEN
+    psn := psn - 1; -- minus-1 from comma position
+    -- RAISE NOTICE 'psn %',psn;
+    argx := SUBSTRING(arg_str FROM 1 FOR psn);  -- get one set member
+    psn := psn + 2; -- go to first starting letter
+    arg_str := SUBSTRING(arg_str FROM psn);   -- hack it off
+ELSE
+    psn := 0; -- minus-1 from comma position
+    argx := arg_str;
+END IF;
+-- RAISE NOTICE 'argx %',argx;
+-- RAISE NOTICE 'new arg_str: %',arg_str;
+WHILE nobreak LOOP
+    EXECUTE 'SELECT count(*) FROM $constraint_table WHERE set_values = ' || quote_literal(argx) INTO rec_count;
+    IF rec_count = 0 THEN RAISE EXCEPTION 'one of the set values was not found';
+    END IF;
+    IF psn > 0 THEN
+        psn := psn - 1; -- minus-1 from comma position
+        -- RAISE NOTICE 'psn %',psn;
+        argx := SUBSTRING(arg_str FROM 1 FOR psn);  -- get one set member
+        psn := psn + 2; -- go to first starting letter
+        arg_str := SUBSTRING(arg_str FROM psn);   -- hack it off
+        psn := POSITION(',' in arg_str);
+    ELSE nobreak = 0;
+    END IF;
+    -- RAISE NOTICE 'next argx % and next arg_str %', argx, arg_str;
+END LOOP;
+RETURN NEW;
+----
+END;
+\$\$ LANGUAGE 'plpgsql' VOLATILE;
+
+drop trigger set_test ON $table;
+-- make a trigger for each set field
+-- make trigger and hard-code in column names
+-- see http://archives.postgresql.org/pgsql-interfaces/2005-02/msg00020.php
+CREATE   TRIGGER    set_test
+BEFORE   INSERT OR   UPDATE  ON $table   FOR  EACH  ROW
+EXECUTE  PROCEDURE  check_".$table."_".$column_name."_set();\n";
+} #  end sub make_plpgsql();
+

nominatim/index.c

@@ -308,6 +308,13 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
     {
         nominatim_exportXMLEnd(writer);
     }
+
+    // Close all connections
+    for (i = 0; i < num_threads; i++)
+    {
+        PQfinish(thread_data[i].conn);
+    }
+    PQfinish(conn);
 }
 
 void *nominatim_indexThread(void * thread_data_in)

settings/settings.php

@@ -4,17 +4,53 @@
 
 	// General settings
 	@define('CONST_Debug', false);
-	@define('CONST_Database_DSN', 'pgsql://@/nominatim');
+	@define('CONST_Database_DSN', 'pgsql://@/nominatim'); // <driver>://<username>:<password>@<host>:<port>/<database>
 	@define('CONST_Max_Word_Frequency', '50000');
 
+	// Software versions
+	@define('CONST_Postgresql_Version', '9.1'); // values: 8.3, 8.4, 9.0, 9.1, 9.2
+	@define('CONST_Postgis_Version', '1.5'); // values: 1.5, 2.0
+
 	// Paths
-	@define('CONST_Postgresql_Version', '9.1');
 	@define('CONST_Path_Postgresql_Contrib', '/usr/share/postgresql/'.CONST_Postgresql_Version.'/contrib');
-	@define('CONST_Path_Postgresql_Postgis', CONST_Path_Postgresql_Contrib.'/postgis-1.5');
+	@define('CONST_Path_Postgresql_Postgis', CONST_Path_Postgresql_Contrib.'/postgis-'.CONST_Postgis_Version);
 	@define('CONST_Osm2pgsql_Binary', CONST_BasePath.'/osm2pgsql/osm2pgsql');
 	@define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
 
+	// osm2pgsql settings
+	@define('CONST_Osm2pgsql_Flatnode_File', null);
+
+	// Replication settings
+	@define('CONST_Replication_Url', 'http://planet.openstreetmap.org/replication/minute');
+	@define('CONST_Replication_MaxInterval', '3600');
+	@define('CONST_Replication_Update_Interval', '60');  // How often upstream publishes diffs
+	@define('CONST_Replication_Recheck_Interval', '60'); // How long to sleep if no update found yet
+
+	// Connection buckets to rate limit people being nasty
+	@define('CONST_ConnectionBucket_MemcacheServerAddress', false);
+	@define('CONST_ConnectionBucket_MemcacheServerPort', 11211);
+	@define('CONST_ConnectionBucket_MaxBlockList', 100);
+	@define('CONST_ConnectionBucket_LeakRate', 1);
+	@define('CONST_ConnectionBucket_BlockLimit', 10);
+	@define('CONST_ConnectionBucket_WaitLimit', 6);
+	@define('CONST_ConnectionBucket_MaxSleeping', 10);
+	@define('CONST_ConnectionBucket_Cost_Reverse', 1);
+	@define('CONST_ConnectionBucket_Cost_Search', 2);
+	@define('CONST_ConnectionBucket_Cost_Details', 3);
+	@define('CONST_ConnectionBucket_Cost_Status', 1);
+
+	// Override this function to add an adjustment factor to the cost
+	// based on server load. e.g. getBlockingProcesses
+	if (!function_exists('user_busy_cost'))
+	{
+		function user_busy_cost()
+		{
+			return 0;
+		}
+	}
+
 	// Website settings
+	@define('CONST_NoAccessControl', true);
 	@define('CONST_ClosedForIndexing', false);
 	@define('CONST_ClosedForIndexingExceptionIPs', '');
 	@define('CONST_BlockedIPs', '');
@@ -23,7 +59,7 @@
 	@define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/');
 	@define('CONST_Tile_Default', 'Mapnik');
 
-	@define('CONST_Default_Language', 'xx');
+	@define('CONST_Default_Language', false);
 	@define('CONST_Default_Lat', 20.0);
 	@define('CONST_Default_Lon', 0.0);
 	@define('CONST_Default_Zoom', 2);
@@ -31,7 +67,13 @@
 	@define('CONST_Search_AreaPolygons_Enabled', true);
 	@define('CONST_Search_AreaPolygons', true);
 
-	@define('CONST_Suggestions_Enabled', false);
+	@define('CONST_Search_BatchMode', false);
+
+	@define('CONST_Search_TryDroppedAddressTerms', false);
+	@define('CONST_Search_NameOnlySearchFrequencyThreshold', false);
+
+	// Set to zero to disable polygon output
+	@define('CONST_PolygonOutput_MaximumTypes', 1);
 
 	// Log settings
 	@define('CONST_Log_DB', true);

sql/country_grid.sql

@@ -1,15 +1,33 @@
-drop table country_osm_grid2;
-create table country_osm_grid2 as select country_name.country_code,st_union(placex.geometry) as geometry from country_name,
+-- Script to build a calculated country grid from existing tables
+DROP TABLE IF EXISTS tmp_country_osm_grid;
+CREATE TABLE tmp_country_osm_grid as select country_name.country_code,st_union(placex.geometry) as geometry from country_name,
   placex
   where (lower(placex.country_code) = country_name.country_code)
-    and placex.rank_search < 16 and st_area(placex.geometry)>0 
+    and placex.rank_search < 16 and st_area(placex.geometry) > 0 
   group by country_name.country_code;
-alter table country_osm_grid2 add column area double precision;
-update country_osm_grid2 set area = st_area(geometry::geography);
-drop table country_osm_grid3;
-create table country_osm_grid3 as select country_code,area,quad_split_geometry(geometry,0.5,20) as geometry from country_osm_grid2;
-drop table country_osm_grid;
-alter table country_osm_grid3 rename to country_osm_grid;
-CREATE INDEX idx_country_osm_grid_geometry ON country_osm_grid USING GIST (geometry);
-update country_osm_grid set area = sum from (select country_code,sum(case when st_area(geometry::geography) = 'NaN' THEN 0 ELSE st_area(geometry::geography) END) 
- from country_osm_grid group by country_code) as x where x.country_code = country_osm_grid.country_code;
+ALTER TABLE tmp_country_osm_grid add column area double precision;
+UPDATE tmp_country_osm_grid set area = st_area(geometry::geography);
+
+-- compare old and new
+select country_code, round, round(log(area)) from (select distinct country_code,round(log(area)) from country_osm_grid order by country_code) as x 
+  left outer join tmp_country_osm_grid using (country_code) where area is null or round(log(area)) != round;
+
+DROP TABLE IF EXISTS new_country_osm_grid;
+CREATE TABLE new_country_osm_grid as select country_code,area,quad_split_geometry(geometry,0.5,20) as geometry from tmp_country_osm_grid;
+CREATE INDEX new_idx_country_osm_grid_geometry ON new_country_osm_grid USING GIST (geometry);
+
+-- Sometimes there are problems calculating area due to invalid data - optionally recalc
+UPDATE new_country_osm_grid set area = sum from (select country_code,sum(case when st_area(geometry::geography) = 'NaN' THEN 0 ELSE st_area(geometry::geography) END) 
+ from new_country_osm_grid group by country_code) as x where x.country_code = new_country_osm_grid.country_code;
+
+-- compare old and new
+select country_code, x.round, y.round from (select distinct country_code,round(log(area)) from country_osm_grid order by country_code) as x
+  left outer join (select distinct country_code,round(log(area)) from new_country_osm_grid order by country_code) as y
+    using (country_code) where x.round != y.round;
+
+-- Flip the new table in
+BEGIN;
+DROP TABLE IF EXISTS country_osm_grid;
+ALTER TABLE new_country_osm_grid rename to country_osm_grid;
+ALTER INDEX new_idx_country_osm_grid_geometry RENAME TO idx_country_osm_grid_geometry;
+COMMIT;

sql/functions.sql

@@ -5,7 +5,6 @@
 --  score integer
 --);
 
-
 CREATE OR REPLACE FUNCTION getclasstypekey(c text, t text) RETURNS TEXT
   AS $$
 DECLARE
@@ -96,7 +95,7 @@ BEGIN
   SELECT min(word_id), max(search_name_count) FROM word WHERE word_token = lookup_token and class is null and type is null into return_word_id, count;
   IF return_word_id IS NULL THEN
     return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, regexp_replace(lookup_token,E'([^0-9])\\1+',E'\\1','g'), null, null, null, null, 0, null);
+    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, null, null, 0);
   ELSE
     IF count > get_maxwordfreq() THEN
       return_word_id := NULL;
@@ -118,7 +117,7 @@ BEGIN
   SELECT min(word_id) FROM word WHERE word_token = lookup_token and class='place' and type='house' into return_word_id;
   IF return_word_id IS NULL THEN
     return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, 'place', 'house', null, 0, null);
+    INSERT INTO word VALUES (return_word_id, lookup_token, null, 'place', 'house', null, 0);
   END IF;
   RETURN return_word_id;
 END;
@@ -136,7 +135,7 @@ BEGIN
   SELECT min(word_id) FROM word WHERE word_token = lookup_token and country_code=lookup_country_code into return_word_id;
   IF return_word_id IS NULL THEN
     return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, null, null, lookup_country_code, 0, null);
+    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, null, lookup_country_code, 0);
   END IF;
   RETURN return_word_id;
 END;
@@ -154,7 +153,7 @@ BEGIN
   SELECT min(word_id) FROM word WHERE word_token = lookup_token and class=lookup_class and type = lookup_type into return_word_id;
   IF return_word_id IS NULL THEN
     return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, lookup_class, lookup_type, null, 0, null);
+    INSERT INTO word VALUES (return_word_id, lookup_token, null, lookup_class, lookup_type, null, 0);
   END IF;
   RETURN return_word_id;
 END;
@@ -172,7 +171,7 @@ BEGIN
   SELECT min(word_id) FROM word WHERE word_token = lookup_token into return_word_id;
   IF return_word_id IS NULL THEN
     return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, null, null, null, 0, null);
+    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, null, null, 0);
   END IF;
   RETURN return_word_id;
 END;
@@ -204,7 +203,7 @@ BEGIN
   SELECT min(word_id) FROM word WHERE word_token = lookup_token and class=lookup_class and type = lookup_type and operator = op into return_word_id;
   IF return_word_id IS NULL THEN
     return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, null, null, lookup_class, lookup_type, null, 0, op, null);
+    INSERT INTO word VALUES (return_word_id, lookup_token, null, lookup_class, lookup_type, null, 0, op);
   END IF;
   RETURN return_word_id;
 END;
@@ -223,7 +222,7 @@ BEGIN
   SELECT min(word_id) FROM word WHERE word_token = lookup_token and class is null and type is null into return_word_id;
   IF return_word_id IS NULL THEN
     return_word_id := nextval('seq_word');
-    INSERT INTO word VALUES (return_word_id, lookup_token, regexp_replace(lookup_token,E'([^0-9])\\1+',E'\\1','g'), src_word, null, null, null, 0, null);
+    INSERT INTO word VALUES (return_word_id, lookup_token, src_word, null, null, null, 0);
 --    nospace_lookup_token := replace(replace(lookup_token, '-',''), ' ','');
 --    IF ' '||nospace_lookup_token != lookup_token THEN
 --      INSERT INTO word VALUES (return_word_id, '-'||nospace_lookup_token, null, src_word, null, null, null, 0, null);
@@ -346,6 +345,12 @@ BEGIN
       result := result || w;
     END IF;
 
+    w := getorcreate_word_id(s);
+
+    IF NOT (ARRAY[w] <@ result) THEN
+      result := result || w;
+    END IF;
+
     words := string_to_array(s, ' ');
     IF array_upper(words, 1) IS NOT NULL THEN
       FOR j IN 1..array_upper(words, 1) LOOP
@@ -408,6 +413,12 @@ BEGIN
     result := result || w;
   END IF;
 
+  w := getorcreate_word_id(s);
+
+  IF NOT (ARRAY[w] <@ result) THEN
+    result := result || w;
+  END IF;
+
   words := string_to_array(s, ' ');
   IF array_upper(words, 1) IS NOT NULL THEN
     FOR j IN 1..array_upper(words, 1) LOOP
@@ -484,6 +495,12 @@ BEGIN
 
 --DEBUG: RAISE WARNING 'get_country_code, start: %', ST_AsText(place_centre);
 
+  -- Try for a OSM polygon
+  FOR nearcountry IN select country_code from location_area_country where country_code is not null and not isguess and st_covers(geometry, place_centre) limit 1
+  LOOP
+    RETURN nearcountry.country_code;
+  END LOOP;
+
 --DEBUG: RAISE WARNING 'osm fallback: %', ST_AsText(place_centre);
 
   -- Try for OSM fallback data
@@ -493,12 +510,6 @@ BEGIN
     RETURN nearcountry.country_code;
   END LOOP;
 
-  -- Try for a OSM polygon
-  FOR nearcountry IN select country_code from location_area_country where country_code is not null and not isguess and st_covers(geometry, place_centre) limit 1
-  LOOP
-    RETURN nearcountry.country_code;
-  END LOOP;
-
 --DEBUG: RAISE WARNING 'natural earth: %', ST_AsText(place_centre);
 
   -- Natural earth data
@@ -571,6 +582,20 @@ END;
 $$
 LANGUAGE plpgsql IMMUTABLE;
 
+CREATE OR REPLACE FUNCTION get_country_language_codes(search_country_code VARCHAR(2)) RETURNS TEXT[]
+  AS $$
+DECLARE
+  nearcountry RECORD;
+BEGIN
+  FOR nearcountry IN select country_default_language_codes from country_name where country_code = search_country_code limit 1
+  LOOP
+    RETURN lower(nearcountry.country_default_language_codes);
+  END LOOP;
+  RETURN NULL;
+END;
+$$
+LANGUAGE plpgsql IMMUTABLE;
+
 CREATE OR REPLACE FUNCTION get_partition(place geometry, in_country_code VARCHAR(10)) RETURNS INTEGER
   AS $$
 DECLARE
@@ -624,7 +649,7 @@ BEGIN
 
 --  RAISE WARNING 'Adding location with rank > 25 (% rank %)', place_id, rank_search;
 
-  x := deleteLocationArea(partition, place_id);
+  x := deleteLocationArea(partition, place_id, rank_search);
 
   isarea := false;
   IF (ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(geometry)) THEN
@@ -691,7 +716,7 @@ CREATE OR REPLACE FUNCTION update_location(
 DECLARE
   b BOOLEAN;
 BEGIN
-  b := deleteLocationArea(partition, place_id);
+  b := deleteLocationArea(partition, place_id, rank_search);
 --  result := add_location(NEW.place_id, NEW.country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, NEW.geometry);
   RETURN add_location(place_id, place_country_code, name, rank_search, rank_address, geometry);
 END;
@@ -769,8 +794,8 @@ CREATE OR REPLACE FUNCTION create_interpolation(wayid BIGINT, interpolationtype
 DECLARE
   
   newpoints INTEGER;
-  waynodes integer[];
-  nodeid INTEGER;
+  waynodes BIGINT[];
+  nodeid BIGINT;
   prevnode RECORD;
   nextnode RECORD;
   startnumber INTEGER;
@@ -798,10 +823,14 @@ BEGIN
 
       FOR nodeidpos in 1..array_upper(waynodes, 1) LOOP
 
-        select min(place_id) from placex where osm_type = 'N' and osm_id = waynodes[nodeidpos]::INTEGER and type = 'house' INTO search_place_id;
+        -- If there is a place of a type other than place/house, use that because
+        -- it is guaranteed to be the original node. For place/house types use the
+        -- one with the smallest id because the original node was created first.
+        -- Ignore all nodes marked for deletion. (Might happen when the type changes.)
+        select place_id from placex where osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT and indexed_status < 100 order by (type = 'house'),place_id limit 1 INTO search_place_id;
         IF search_place_id IS NULL THEN
-          -- null record of right type
-          select * from placex where osm_type = 'N' and osm_id = waynodes[nodeidpos]::INTEGER and type = 'house' limit 1 INTO nextnode;
+          -- if no such node exists, create a record of the right type
+          select * from placex where osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT and type = 'house' limit 1 INTO nextnode;
           select ST_SetSRID(ST_Point(lon::float/10000000,lat::float/10000000),4326) from planet_osm_nodes where id = waynodes[nodeidpos] INTO nextnode.geometry;
           IF nextnode.geometry IS NULL THEN
             -- we don't have any information about this point, most likely
@@ -863,9 +892,9 @@ BEGIN
               FOR housenum IN startnumber..endnumber BY stepsize LOOP
                 -- this should really copy postcodes but it puts a huge burdon on the system for no big benefit
                 -- ideally postcodes should move up to the way
-                insert into placex (osm_type, osm_id, class, type, admin_level, housenumber, street, isin, postcode,
+                insert into placex (osm_type, osm_id, class, type, admin_level, housenumber, street, addr_place, isin, postcode,
                   country_code, parent_place_id, rank_address, rank_search, indexed_status, geometry)
-                  values ('N',prevnode.osm_id, prevnode.class, prevnode.type, prevnode.admin_level, housenum, prevnode.street, prevnode.isin, coalesce(prevnode.postcode, defpostalcode),
+                  values ('N',prevnode.osm_id, 'place', 'house', prevnode.admin_level, housenum, prevnode.street, prevnode.addr_place, prevnode.isin, coalesce(prevnode.postcode, defpostalcode),
                   prevnode.country_code, prevnode.parent_place_id, prevnode.rank_address, prevnode.rank_search, 1, ST_Line_Interpolate_Point(linegeo, (housenum::float-orginalstartnumber::float)/originalnumberrange::float));
                 newpoints := newpoints + 1;
 --RAISE WARNING 'interpolation number % % ',prevnode.place_id,housenum;
@@ -909,10 +938,6 @@ BEGIN
   --DEBUG: RAISE WARNING '% %',NEW.osm_type,NEW.osm_id;
 
   -- just block these
-  IF NEW.class = 'highway' and NEW.type in ('turning_circle','traffic_signals','mini_roundabout','noexit','crossing') THEN
---    RAISE WARNING 'bad highway %',NEW.osm_id;
-    RETURN null;
-  END IF;
   IF NEW.class in ('landuse','natural') and NEW.name is null THEN
 --    RAISE WARNING 'empty landuse %',NEW.osm_id;
     RETURN null;
@@ -950,9 +975,9 @@ BEGIN
     default_language := get_country_language_code(NEW.calculated_country_code);
     IF default_language IS NOT NULL THEN
       IF NEW.name ? 'name' AND NOT NEW.name ? ('name:'||default_language) THEN
-        NEW.name := NEW.name || (('name:'||default_language) => (NEW.name -> 'name'));
+        NEW.name := NEW.name || hstore(('name:'||default_language), (NEW.name -> 'name'));
       ELSEIF NEW.name ? ('name:'||default_language) AND NOT NEW.name ? 'name' THEN
-        NEW.name := NEW.name || ('name' => (NEW.name -> ('name:'||default_language)));
+        NEW.name := NEW.name || hstore('name', (NEW.name -> ('name:'||default_language)));
       END IF;
     END IF;
   END IF;
@@ -979,7 +1004,7 @@ BEGIN
             RETURN NULL;
         END IF;
 
-        NEW.name := 'ref'=>NEW.postcode;
+        NEW.name := hstore('ref', NEW.postcode);
 
         IF NEW.calculated_country_code = 'gb' THEN
 
@@ -1065,10 +1090,10 @@ BEGIN
       ELSEIF NEW.type in ('national_park') THEN
         NEW.rank_search := 18;
         NEW.rank_address := 18;
-      ELSEIF NEW.type in ('suburb','croft','subdivision') THEN
+      ELSEIF NEW.type in ('suburb','croft','subdivision','isolated_dwelling') THEN
         NEW.rank_search := 20;
         NEW.rank_address := NEW.rank_search;
-      ELSEIF NEW.type in ('farm','locality','islet','isolated_dwelling','mountain_pass') THEN
+      ELSEIF NEW.type in ('farm','locality','islet','mountain_pass') THEN
         NEW.rank_search := 20;
         NEW.rank_address := 0;
         -- Irish townlands, tagged as place=locality and locality=townland
@@ -1130,6 +1155,9 @@ BEGIN
       NEW.rank_address := NEW.rank_search;
     ELSEIF NEW.class = 'natural' and NEW.type in ('coastline') THEN
       RETURN NULL;
+    ELSEIF NEW.class = 'mountain_pass' THEN
+        NEW.rank_search := 20;
+        NEW.rank_address := 0;
     END IF;
 
   END IF;
@@ -1168,9 +1196,9 @@ BEGIN
 
       -- work around bug in postgis, this may have been fixed in 2.0.0 (see http://trac.osgeo.org/postgis/ticket/547)
       update placex set indexed_status = 2 where (st_covers(NEW.geometry, placex.geometry) OR ST_Intersects(NEW.geometry, placex.geometry)) 
-       AND rank_search > NEW.rank_search and indexed_status = 0 and ST_geometrytype(placex.geometry) = 'ST_Point' and (rank_search < 28 or name is not null);
+       AND rank_search > NEW.rank_search and indexed_status = 0 and ST_geometrytype(placex.geometry) = 'ST_Point' and (rank_search < 28 or name is not null or (NEW.rank_search >= 16 and addr_place is not null));
       update placex set indexed_status = 2 where (st_covers(NEW.geometry, placex.geometry) OR ST_Intersects(NEW.geometry, placex.geometry)) 
-       AND rank_search > NEW.rank_search and indexed_status = 0 and ST_geometrytype(placex.geometry) != 'ST_Point' and (rank_search < 28 or name is not null);
+       AND rank_search > NEW.rank_search and indexed_status = 0 and ST_geometrytype(placex.geometry) != 'ST_Point' and (rank_search < 28 or name is not null or (NEW.rank_search >= 16 and addr_place is not null));
     END IF;
   ELSE
     -- mark nearby items for re-indexing, where 'nearby' depends on the features rank_search and is a complete guess :(
@@ -1195,7 +1223,16 @@ BEGIN
     END IF;
     IF diameter > 0 THEN
 --      RAISE WARNING 'placex point insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,diameter;
-      update placex set indexed_status = 2 where indexed_status = 0 and rank_search > NEW.rank_search and ST_DWithin(placex.geometry, NEW.geometry, diameter) and (rank_search < 28 or name is not null);
+      IF NEW.rank_search >= 26 THEN
+        -- roads may cause reparenting for >27 rank places
+        update placex set indexed_status = 2 where indexed_status = 0 and rank_search > NEW.rank_search and ST_DWithin(placex.geometry, NEW.geometry, diameter);
+      ELSEIF NEW.rank_search >= 16 THEN
+        -- up to rank 16, street-less addresses may need reparenting
+        update placex set indexed_status = 2 where indexed_status = 0 and rank_search > NEW.rank_search and ST_DWithin(placex.geometry, NEW.geometry, diameter) and (rank_search < 28 or name is not null or addr_place is not null);
+      ELSE
+        -- for all other places the search terms may change as well
+        update placex set indexed_status = 2 where indexed_status = 0 and rank_search > NEW.rank_search and ST_DWithin(placex.geometry, NEW.geometry, diameter) and (rank_search < 28 or name is not null);
+      END IF;
     END IF;
 
   END IF;
@@ -1275,7 +1312,7 @@ BEGIN
     RETURN NULL;
   END IF;
 
-  IF NEW.indexed_status != 0 OR OLD.indexed_status = 0 OR NEW.linked_place_id is not null THEN
+  IF NEW.indexed_status != 0 OR OLD.indexed_status = 0 THEN
     RETURN NEW;
   END IF;
 
@@ -1289,25 +1326,27 @@ BEGIN
     RETURN NEW;
   END IF;
 
+  -- TODO: this test is now redundant?
   IF OLD.indexed_status != 0 THEN
-    --DEBUG: RAISE WARNING 'placex_update_0 % %',NEW.osm_type,NEW.osm_id;
 
     NEW.indexed_date = now();
 
+    result := deleteSearchName(NEW.partition, NEW.place_id);
+    DELETE FROM place_addressline WHERE place_id = NEW.place_id;
+    DELETE FROM place_boundingbox where place_id = NEW.place_id;
+    result := deleteRoad(NEW.partition, NEW.place_id);
+    result := deleteLocationArea(NEW.partition, NEW.place_id, NEW.rank_search);
+    UPDATE placex set linked_place_id = null where linked_place_id = NEW.place_id;
+
+    IF NEW.linked_place_id is not null THEN
+      RETURN NEW;
+    END IF;
+
     IF NEW.class = 'place' AND NEW.type = 'houses' THEN
       i := create_interpolation(NEW.osm_id, NEW.housenumber);
       RETURN NEW;
     END IF;
 
-    IF OLD.indexed_status > 0 THEN
-      result := deleteSearchName(NEW.partition, NEW.place_id);
-      DELETE FROM place_addressline WHERE place_id = NEW.place_id;
-      DELETE FROM place_boundingbox where place_id = NEW.place_id;
-      result := deleteRoad(NEW.partition, NEW.place_id);
-      result := deleteLocationArea(NEW.partition, NEW.place_id);
-      UPDATE placex set linked_place_id = null where linked_place_id = NEW.place_id;
-    END IF;
-
     -- Speed up searches - just use the centroid of the feature
     -- cheaper but less acurate
     place_centroid := ST_PointOnSurface(NEW.geometry);
@@ -1335,9 +1374,9 @@ BEGIN
       default_language := get_country_language_code(NEW.calculated_country_code);
       IF default_language IS NOT NULL THEN
         IF NEW.name ? 'name' AND NOT NEW.name ? ('name:'||default_language) THEN
-          NEW.name := NEW.name || (('name:'||default_language) => (NEW.name -> 'name'));
+          NEW.name := NEW.name || hstore(('name:'||default_language), (NEW.name -> 'name'));
         ELSEIF NEW.name ? ('name:'||default_language) AND NOT NEW.name ? 'name' THEN
-          NEW.name := NEW.name || ('name' => (NEW.name -> ('name:'||default_language)));
+          NEW.name := NEW.name || hstore('name', (NEW.name -> ('name:'||default_language)));
         END IF;
       END IF;
     END IF;
@@ -1369,6 +1408,9 @@ BEGIN
 
 --RAISE WARNING 'finding street for %', NEW;
 
+      -- We won't get a better centroid, besides these places are too small to care
+      NEW.centroid := place_centroid;
+
       NEW.parent_place_id := null;
 
       -- to do that we have to find our parent road
@@ -1384,7 +1426,7 @@ BEGIN
             FOR i IN 1..array_upper(relation.members, 1) BY 2 LOOP
               IF NEW.parent_place_id IS NULL AND relation.members[i+1] = 'street' THEN
 --RAISE WARNING 'node in relation %',relation;
-                SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::integer 
+                SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::bigint 
                   and rank_search = 26 INTO NEW.parent_place_id;
               END IF;
             END LOOP;
@@ -1413,7 +1455,7 @@ BEGIN
                   FOR i IN 1..array_upper(relation.members, 1) BY 2 LOOP
                     IF NEW.parent_place_id IS NULL AND relation.members[i+1] = 'street' THEN
     --RAISE WARNING 'node in way that is in a relation %',relation;
-                      SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::integer 
+                      SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::bigint 
                         and rank_search = 26 INTO NEW.parent_place_id;
                     END IF;
                   END LOOP;
@@ -1422,13 +1464,18 @@ BEGIN
           END IF;    
           
           -- If the way contains an explicit name of a street copy it
-          IF NEW.street IS NULL AND location.street IS NOT NULL THEN
+          IF NEW.street IS NULL AND NEW.addr_place IS NULL AND location.street IS NOT NULL THEN
 --RAISE WARNING 'node in way that has a streetname %',location;
             NEW.street := location.street;
           END IF;
 
+          -- IF the way contains an explicit name of a place copy it
+          IF NEW.addr_place IS NULL AND NEW.street IS NULL AND location.addr_place IS NOT NULL THEN
+            NEW.addr_place := location.addr_place;
+          END IF;
+
           -- If this way is a street interpolation line then it is probably as good as we are going to get
-          IF NEW.parent_place_id IS NULL AND NEW.street IS NULL AND location.class = 'place' and location.type='houses' THEN
+          IF NEW.parent_place_id IS NULL AND NEW.street IS NULL AND NEW.addr_place IS NULL AND location.class = 'place' and location.type='houses' THEN
             -- Try and find a way that is close roughly parellel to this line
             FOR relation IN SELECT place_id FROM placex
               WHERE ST_DWithin(location.geometry, placex.geometry, 0.001) and placex.rank_search = 26
@@ -1458,7 +1505,7 @@ BEGIN
             FOR i IN 1..array_upper(relation.members, 1) BY 2 LOOP
               IF NEW.parent_place_id IS NULL AND relation.members[i+1] = 'street' THEN
 --RAISE WARNING 'way that is in a relation %',relation;
-                SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::integer
+                SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::bigint
                   and rank_search = 26 INTO NEW.parent_place_id;
               END IF;
             END LOOP;
@@ -1469,9 +1516,18 @@ BEGIN
 --RAISE WARNING 'x3 %',NEW.parent_place_id;
 
       IF NEW.parent_place_id IS NULL AND NEW.street IS NOT NULL THEN
-      	address_street_word_id := get_name_id(make_standard_name(NEW.street));
+        address_street_word_id := get_name_id(make_standard_name(NEW.street));
         IF address_street_word_id IS NOT NULL THEN
           FOR location IN SELECT * from getNearestNamedRoadFeature(NEW.partition, place_centroid, address_street_word_id) LOOP
+              NEW.parent_place_id := location.place_id;
+          END LOOP;
+        END IF;
+      END IF;
+
+      IF NEW.parent_place_id IS NULL AND NEW.addr_place IS NOT NULL THEN
+        address_street_word_id := get_name_id(make_standard_name(NEW.addr_place));
+        IF address_street_word_id IS NOT NULL THEN
+          FOR location IN SELECT * from getNearestNamedPlaceFeature(NEW.partition, place_centroid, address_street_word_id) LOOP
             NEW.parent_place_id := location.place_id;
           END LOOP;
         END IF;
@@ -1512,6 +1568,7 @@ BEGIN
 
         -- Merge address from parent
         nameaddress_vector := array_merge(nameaddress_vector, location.nameaddress_vector);
+        nameaddress_vector := array_merge(nameaddress_vector, location.name_vector);
 --return NEW;
         -- Performance, it would be more acurate to do all the rest of the import process but it takes too long
         -- Just be happy with inheriting from parent road only
@@ -1520,7 +1577,7 @@ BEGIN
           result := add_location(NEW.place_id, NEW.calculated_country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, NEW.geometry);
         END IF;
 
-        result := insertSearchName(NEW.partition, NEW.place_id, NEW.calculated_country_code, name_vector, nameaddress_vector, NEW.rank_search, NEW.rank_address, NEW.importance, place_centroid);
+        result := insertSearchName(NEW.partition, NEW.place_id, NEW.calculated_country_code, name_vector, nameaddress_vector, NEW.rank_search, NEW.rank_address, NEW.importance, place_centroid, NEW.geometry);
 
         return NEW;
       END IF;
@@ -1550,7 +1607,7 @@ BEGIN
             -- merge in the label name, re-init word vector
             IF NOT linkedPlacex.name IS NULL THEN
               NEW.name := linkedPlacex.name || NEW.name;
-              name_vector := make_keywords(NEW.name);
+              name_vector := array_merge(name_vector, make_keywords(linkedPlacex.name));
             END IF;
 
             -- merge in extra tags
@@ -1558,6 +1615,10 @@ BEGIN
               NEW.extratags := linkedPlacex.extratags || NEW.extratags;
             END IF;
 
+            IF NOT NEW.extratags ? linkedPlacex.class THEN
+              NEW.extratags := NEW.extratags || hstore(linkedPlacex.class, linkedPlacex.type);
+            END IF;
+
             -- mark the linked place (excludes from search results)
             UPDATE placex set linked_place_id = NEW.place_id where place_id = linkedPlacex.place_id;
 
@@ -1577,7 +1638,6 @@ BEGIN
               IF make_standard_name(NEW.name->'name') = make_standard_name(linkedPlacex.name->'name') 
                 AND NEW.rank_address = linkedPlacex.rank_address THEN
 
-
                 -- If we don't already have one use this as the centre point of the geometry
                 IF NEW.centroid IS NULL THEN
                   NEW.centroid := coalesce(linkedPlacex.centroid,st_centroid(linkedPlacex.geometry));
@@ -1594,6 +1654,10 @@ BEGIN
                   NEW.extratags := linkedPlacex.extratags || NEW.extratags;
                 END IF;
 
+                IF NOT NEW.extratags ? linkedPlacex.class THEN
+                  NEW.extratags := NEW.extratags || hstore(linkedPlacex.class, linkedPlacex.type);
+                END IF;
+
                 -- mark the linked place (excludes from search results)
                 UPDATE placex set linked_place_id = NEW.place_id where place_id = linkedPlacex.place_id;
 
@@ -1608,6 +1672,11 @@ BEGIN
         END IF;
       END IF;
 
+    END IF;
+
+    -- Name searches can be done for ways as well as relations
+    IF NEW.osm_type in ('W','R') AND NEW.rank_search < 26 THEN
+
       -- not found one yet? how about doing a name search
       IF NEW.centroid IS NULL AND (NEW.name->'name') is not null and make_standard_name(NEW.name->'name') != '' THEN
 
@@ -1631,6 +1700,10 @@ BEGIN
           -- merge in extra tags
           NEW.extratags := linkedPlacex.extratags || NEW.extratags;
 
+          IF NOT NEW.extratags ? linkedPlacex.class THEN
+            NEW.extratags := NEW.extratags || hstore(linkedPlacex.class, linkedPlacex.type);
+          END IF;
+
           -- mark the linked place (excludes from search results)
           UPDATE placex set linked_place_id = NEW.place_id where place_id = linkedPlacex.place_id;
 
@@ -1641,6 +1714,19 @@ BEGIN
 
       IF NEW.centroid IS NOT NULL THEN
         place_centroid := NEW.centroid;
+        -- Place might have had only a name tag before but has now received translations
+        -- from the linked place. Make sure a name tag for the default language exists in
+        -- this case. 
+        IF NEW.name is not null AND array_upper(akeys(NEW.name),1) > 1 THEN
+          default_language := get_country_language_code(NEW.calculated_country_code);
+          IF default_language IS NOT NULL THEN
+            IF NEW.name ? 'name' AND NOT NEW.name ? ('name:'||default_language) THEN
+              NEW.name := NEW.name || hstore(('name:'||default_language), (NEW.name -> 'name'));
+            ELSEIF NEW.name ? ('name:'||default_language) AND NOT NEW.name ? 'name' THEN
+              NEW.name := NEW.name || hstore('name', (NEW.name -> ('name:'||default_language)));
+            END IF;
+          END IF;
+        END IF;
       END IF;
 
       -- Did we gain a wikipedia tag in the process? then we need to recalculate our importance
@@ -1670,8 +1756,15 @@ BEGIN
         FOR i IN 1..array_upper(isin, 1) LOOP
           address_street_word_id := get_name_id(make_standard_name(isin[i]));
           IF address_street_word_id IS NOT NULL AND NOT(ARRAY[address_street_word_id] <@ isin_tokens) THEN
+            nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
             isin_tokens := isin_tokens || address_street_word_id;
           END IF;
+
+          -- merge word into address vector
+          address_street_word_id := get_word_id(make_standard_name(isin[i]));
+          IF address_street_word_id IS NOT NULL THEN
+            nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
+          END IF;
         END LOOP;
       END IF;
     END IF;
@@ -1681,11 +1774,32 @@ BEGIN
         FOR i IN 1..array_upper(isin, 1) LOOP
           address_street_word_id := get_name_id(make_standard_name(isin[i]));
           IF address_street_word_id IS NOT NULL AND NOT(ARRAY[address_street_word_id] <@ isin_tokens) THEN
+            nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
             isin_tokens := isin_tokens || address_street_word_id;
           END IF;
+
+          -- merge into address vector
+          address_street_word_id := get_word_id(make_standard_name(isin[i]));
+          IF address_street_word_id IS NOT NULL THEN
+            nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
+          END IF;
         END LOOP;
       END IF;
     END IF;
+
+    -- for the USA we have an additional address table.  Merge in zip codes from there too
+    IF NEW.rank_search = 26 AND NEW.calculated_country_code = 'us' THEN
+      FOR location IN SELECT distinct postcode from location_property_tiger where parent_place_id = NEW.place_id LOOP
+        address_street_word_id := get_name_id(make_standard_name(location.postcode));
+        nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
+        isin_tokens := isin_tokens || address_street_word_id;
+
+        -- also merge in the single word version
+        address_street_word_id := get_word_id(make_standard_name(location.postcode));
+        nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
+      END LOOP;
+    END IF;
+
 -- RAISE WARNING 'ISIN: %', isin_tokens;
 
     -- Process area matches
@@ -1719,19 +1833,21 @@ BEGIN
         INSERT INTO place_addressline VALUES (NEW.place_id, location.place_id, true, location_isaddress, location.distance, location.rank_address);
 
         IF location_isaddress THEN
-            address_havelevel[location.rank_address] := true;
-            IF NOT location.isguess THEN
-                SELECT geometry FROM placex WHERE place_id = location.place_id INTO location_parent;
-            END IF;
+
+          address_havelevel[location.rank_address] := true;
+          IF NOT location.isguess THEN
+            SELECT geometry FROM placex WHERE place_id = location.place_id INTO location_parent;
+          END IF;
+
+          IF location.rank_address > parent_place_id_rank THEN
+            NEW.parent_place_id = location.place_id;
+            parent_place_id_rank = location.rank_address;
+          END IF;
+
         END IF;
 
 --RAISE WARNING '  Terms: (%) %',location, nameaddress_vector;
 
-        IF location.rank_address > parent_place_id_rank THEN
-          NEW.parent_place_id = location.place_id;
-          parent_place_id_rank = location.rank_address;
-        END IF;
-
       END IF;
 
     END LOOP;
@@ -1799,9 +1915,8 @@ BEGIN
         result := insertLocationRoad(NEW.partition, NEW.place_id, NEW.calculated_country_code, NEW.geometry);
       END IF;
 
-      result := insertSearchName(NEW.partition, NEW.place_id, NEW.calculated_country_code, name_vector, nameaddress_vector, NEW.rank_search, NEW.rank_address, NEW.importance, place_centroid);
+      result := insertSearchName(NEW.partition, NEW.place_id, NEW.calculated_country_code, name_vector, nameaddress_vector, NEW.rank_search, NEW.rank_address, NEW.importance, place_centroid, NEW.geometry);
 
---      INSERT INTO search_name values (NEW.place_id, NEW.rank_search, NEW.rank_search, 0, NEW.calculated_country_code, name_vector, nameaddress_vector, place_centroid);
     END IF;
 
     -- If we've not managed to pick up a better one - default centroid
@@ -1851,7 +1966,7 @@ BEGIN
   --DEBUG: RAISE WARNING 'placex_delete:08 % %',OLD.osm_type,OLD.osm_id;
 
   IF OLD.rank_address < 26 THEN
-    b := deleteLocationArea(OLD.partition, OLD.place_id);
+    b := deleteLocationArea(OLD.partition, OLD.place_id, OLD.rank_search);
   END IF;
 
   --DEBUG: RAISE WARNING 'placex_delete:09 % %',OLD.osm_type,OLD.osm_id;
@@ -1925,9 +2040,6 @@ BEGIN
   END IF;
 
   -- Just block these - lots and pointless
-  IF NEW.class = 'highway' and NEW.type in ('turning_circle','traffic_signals','mini_roundabout','noexit','crossing') THEN
-    RETURN null;
-  END IF;
   IF NEW.class in ('landuse','natural') and NEW.name is null THEN
     RETURN null;
   END IF;
@@ -1978,12 +2090,15 @@ BEGIN
   IF existingplacex.osm_type IS NULL THEN
 
     IF existing.osm_type IS NOT NULL THEN
+      -- pathological case caused by the triggerless copy into place during initial import
+      -- force delete even for large areas, it will be reinserted later
+      UPDATE place set geometry = ST_SetSRID(ST_Point(0,0), 4326) where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
       DELETE from place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
     END IF;
 
     -- No - process it as a new insertion (hopefully of low rank or it will be slow)
     insert into placex (osm_type, osm_id, class, type, name, admin_level, housenumber, 
-      street, isin, postcode, country_code, extratags, geometry)
+      street, addr_place, isin, postcode, country_code, extratags, geometry)
       values (NEW.osm_type
         ,NEW.osm_id
         ,NEW.class
@@ -1992,6 +2107,7 @@ BEGIN
         ,NEW.admin_level
         ,NEW.housenumber
         ,NEW.street
+        ,NEW.addr_place
         ,NEW.isin
         ,NEW.postcode
         ,NEW.country_code
@@ -2017,6 +2133,9 @@ BEGIN
     IF coalesce(existing.street, '') != coalesce(NEW.street, '') THEN
       RAISE WARNING 'update details, street: % % % %',NEW.osm_type,NEW.osm_id,existing.street,NEW.street;
     END IF;
+    IF coalesce(existing.addr_place, '') != coalesce(NEW.addr_place, '') THEN
+      RAISE WARNING 'update details, street: % % % %',NEW.osm_type,NEW.osm_id,existing.addr_place,NEW.addr_place;
+    END IF;
     IF coalesce(existing.isin, '') != coalesce(NEW.isin, '') THEN
       RAISE WARNING 'update details, isin: % % % %',NEW.osm_type,NEW.osm_id,existing.isin,NEW.isin;
     END IF;
@@ -2059,6 +2178,7 @@ BEGIN
   IF FALSE AND existingplacex.rank_search < 26
      AND coalesce(existing.housenumber, '') = coalesce(NEW.housenumber, '')
      AND coalesce(existing.street, '') = coalesce(NEW.street, '')
+     AND coalesce(existing.addr_place, '') = coalesce(NEW.addr_place, '')
      AND coalesce(existing.isin, '') = coalesce(NEW.isin, '')
      AND coalesce(existing.postcode, '') = coalesce(NEW.postcode, '')
      AND coalesce(existing.country_code, '') = coalesce(NEW.country_code, '')
@@ -2081,6 +2201,7 @@ BEGIN
     IF coalesce(existing.name::text, '') != coalesce(NEW.name::text, '')
         OR coalesce(existing.housenumber, '') != coalesce(NEW.housenumber, '')
         OR coalesce(existing.street, '') != coalesce(NEW.street, '')
+        OR coalesce(existing.addr_place, '') != coalesce(NEW.addr_place, '')
         OR coalesce(existing.isin, '') != coalesce(NEW.isin, '')
         OR coalesce(existing.postcode, '') != coalesce(NEW.postcode, '')
         OR coalesce(existing.country_code, '') != coalesce(NEW.country_code, '') THEN
@@ -2099,6 +2220,7 @@ BEGIN
      OR coalesce(existing.extratags::text, '') != coalesce(NEW.extratags::text, '')
      OR coalesce(existing.housenumber, '') != coalesce(NEW.housenumber, '')
      OR coalesce(existing.street, '') != coalesce(NEW.street, '')
+     OR coalesce(existing.addr_place, '') != coalesce(NEW.addr_place, '')
      OR coalesce(existing.isin, '') != coalesce(NEW.isin, '')
      OR coalesce(existing.postcode, '') != coalesce(NEW.postcode, '')
      OR coalesce(existing.country_code, '') != coalesce(NEW.country_code, '')
@@ -2110,10 +2232,12 @@ BEGIN
       name = NEW.name,
       housenumber  = NEW.housenumber,
       street = NEW.street,
+      addr_place = NEW.addr_place,
       isin = NEW.isin,
       postcode = NEW.postcode,
       country_code = NEW.country_code,
       extratags = NEW.extratags,
+      admin_level = NEW.admin_level,
       geometry = NEW.geometry
       where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
 
@@ -2121,11 +2245,13 @@ BEGIN
       name = NEW.name,
       housenumber = NEW.housenumber,
       street = NEW.street,
+      addr_place = NEW.addr_place,
       isin = NEW.isin,
       postcode = NEW.postcode,
       country_code = NEW.country_code,
       parent_place_id = null,
       extratags = NEW.extratags,
+      admin_level = CASE WHEN NEW.admin_level > 15 THEN 15 ELSE NEW.admin_level END,
       indexed_status = 2,    
       geometry = NEW.geometry
       where place_id = existingplacex.place_id;
@@ -2157,7 +2283,9 @@ BEGIN
     END IF;
   END LOOP;
 
-  RETURN null;
+  -- anything will do as a fallback - just take the first name type thing there is
+  search := avals(name);
+  RETURN search[1];
 END;
 $$
 LANGUAGE plpgsql IMMUTABLE;
@@ -2250,11 +2378,11 @@ END;
 $$
 LANGUAGE plpgsql;
 
-DROP TYPE addressline CASCADE;
+DROP TYPE IF EXISTS addressline CASCADE;
 create type addressline as (
   place_id BIGINT,
   osm_type CHAR(1),
-  osm_id INTEGER,
+  osm_id BIGINT,
   name HSTORE,
   class TEXT,
   type TEXT,
@@ -2313,7 +2441,7 @@ BEGIN
   hadcountry := false;
   FOR location IN 
     select placex.place_id, osm_type, osm_id,
-      CASE WHEN class = 'place' and type = 'postcode' THEN 'name' => postcode ELSE name END as name,
+      CASE WHEN class = 'place' and type = 'postcode' THEN hstore('name', postcode) ELSE name END as name,
       class, type, admin_level, true as fromarea, true as isaddress,
       CASE WHEN rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address,
       0 as distance, calculated_country_code
@@ -2346,7 +2474,7 @@ BEGIN
 
   FOR location IN 
     select placex.place_id, osm_type, osm_id,
-      CASE WHEN class = 'place' and type = 'postcode' THEN 'name' => postcode ELSE name END as name,
+      CASE WHEN class = 'place' and type = 'postcode' THEN hstore('name', postcode) ELSE name END as name,
       class, type, admin_level, fromarea, isaddress,
       CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address,
       distance,calculated_country_code
@@ -2391,7 +2519,7 @@ BEGIN
   END IF;
 
   IF searchcountrycode IS NOT NULL THEN
-    location := ROW(null, null, null, 'ref'=>searchcountrycode, 'place', 'country_code', null, true, false, 4, 0)::addressline;
+    location := ROW(null, null, null, hstore('ref', searchcountrycode), 'place', 'country_code', null, true, false, 4, 0)::addressline;
     RETURN NEXT location;
   END IF;
 
@@ -2402,12 +2530,12 @@ BEGIN
   END IF;
 
   IF searchhousenumber IS NOT NULL THEN
-    location := ROW(in_place_id, null, null, 'ref'=>searchhousenumber, 'place', 'house_number', null, true, true, 28, 0)::addressline;
+    location := ROW(in_place_id, null, null, hstore('ref', searchhousenumber), 'place', 'house_number', null, true, true, 28, 0)::addressline;
     RETURN NEXT location;
   END IF;
 
   IF searchpostcode IS NOT NULL THEN
-    location := ROW(null, null, null, 'ref'=>searchpostcode, 'place', 'postcode', null, true, true, 5, 0)::addressline;
+    location := ROW(null, null, null, hstore('ref', searchpostcode), 'place', 'postcode', null, true, true, 5, 0)::addressline;
     RETURN NEXT location;
   END IF;
 
@@ -2502,6 +2630,7 @@ BEGIN
       name = place.name,
       housenumber = place.housenumber,
       street = place.street,
+      addr_place = place.addr_place,
       isin = place.isin,
       postcode = place.postcode,
       country_code = place.country_code,
@@ -2903,3 +3032,79 @@ BEGIN
 END;
 $$
 LANGUAGE plpgsql;
+
+
+CREATE OR REPLACE FUNCTION place_force_delete(placeid BIGINT) RETURNS BOOLEAN
+  AS $$
+DECLARE
+    osmid BIGINT;
+    osmtype character(1);
+    pclass text;
+    ptype text;
+BEGIN
+  SELECT osm_type, osm_id, class, type FROM placex WHERE place_id = placeid INTO osmtype, osmid, pclass, ptype;
+  DELETE FROM import_polygon_delete where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
+  DELETE FROM import_polygon_error where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
+  -- force delete from place/placex by making it a very small geometry
+  UPDATE place set geometry = ST_SetSRID(ST_Point(0,0), 4326) where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
+  DELETE FROM place where osm_type = osmtype and osm_id = osmid and class = pclass and type = ptype;
+
+  RETURN TRUE;
+END;
+$$
+LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION place_force_update(placeid BIGINT) RETURNS BOOLEAN
+  AS $$
+DECLARE
+  placegeom GEOMETRY;
+  geom GEOMETRY;
+  diameter FLOAT;
+  rank INTEGER;
+BEGIN
+  SELECT geometry, rank_search FROM placex WHERE place_id = placeid INTO placegeom, rank;
+  IF placegeom IS NOT NULL AND ST_IsValid(placegeom) THEN
+    IF ST_GeometryType(placegeom) in ('ST_Polygon','ST_MultiPolygon') THEN
+      FOR geom IN select split_geometry(placegeom) FROM placex WHERE place_id = placeid LOOP
+        update placex set indexed_status = 2 where (st_covers(geom, placex.geometry) OR ST_Intersects(geom, placex.geometry)) 
+        AND rank_search > rank and indexed_status = 0 and ST_geometrytype(placex.geometry) = 'ST_Point' and (rank_search < 28 or name is not null or (rank >= 16 and addr_place is not null));
+        update placex set indexed_status = 2 where (st_covers(geom, placex.geometry) OR ST_Intersects(geom, placex.geometry)) 
+        AND rank_search > rank and indexed_status = 0 and ST_geometrytype(placex.geometry) != 'ST_Point' and (rank_search < 28 or name is not null or (rank >= 16 and addr_place is not null));
+      END LOOP;
+    ELSE
+        diameter := 0;
+        IF rank = 11 THEN
+          diameter := 0.05;
+        ELSEIF rank < 18 THEN
+          diameter := 0.1;
+        ELSEIF rank < 20 THEN
+          diameter := 0.05;
+        ELSEIF rank = 21 THEN
+          diameter := 0.001;
+        ELSEIF rank < 24 THEN
+          diameter := 0.02;
+        ELSEIF rank < 26 THEN
+          diameter := 0.002; -- 100 to 200 meters
+        ELSEIF rank < 28 THEN
+          diameter := 0.001; -- 50 to 100 meters
+        END IF;
+        IF diameter > 0 THEN
+          IF rank >= 26 THEN
+            -- roads may cause reparenting for >27 rank places
+            update placex set indexed_status = 2 where indexed_status = 0 and rank_search > rank and ST_DWithin(placex.geometry, placegeom, diameter);
+          ELSEIF rank >= 16 THEN
+            -- up to rank 16, street-less addresses may need reparenting
+            update placex set indexed_status = 2 where indexed_status = 0 and rank_search > rank and ST_DWithin(placex.geometry, placegeom, diameter) and (rank_search < 28 or name is not null or addr_place is not null);
+          ELSE
+            -- for all other places the search terms may change as well
+            update placex set indexed_status = 2 where indexed_status = 0 and rank_search > rank and ST_DWithin(placex.geometry, placegeom, diameter) and (rank_search < 28 or name is not null);
+          END IF;
+        END IF;
+    END IF;
+    RETURN TRUE;
+  END IF;
+
+  RETURN FALSE;
+END;
+$$
+LANGUAGE plpgsql;

sql/hstore_compatability_9_0.sql

@@ -0,0 +1,8 @@
+CREATE OR REPLACE FUNCTION hstore(k text, v text) RETURNS HSTORE
+  AS $$
+DECLARE
+BEGIN
+  RETURN k => v;
+END;
+$$
+LANGUAGE plpgsql IMMUTABLE;

sql/indices.src.sql

@@ -21,7 +21,6 @@ CREATE INDEX idx_placex_interpolation ON placex USING BTREE (geometry_sector) wh
 CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id);
 
 CREATE INDEX idx_search_name_country_centroid ON search_name_country USING GIST (centroid);
-CREATE INDEX idx_search_name_country_nameaddress_vector ON search_name_country USING GIN (nameaddress_vector) WITH (fastupdate = off);
 
 -- start
 CREATE INDEX idx_location_property_-partition-_centroid ON location_property_-partition- USING GIST (centroid);

sql/partition-functions.src.sql

@@ -1,62 +1,3 @@
-drop type nearplace cascade;
-create type nearplace as (
-  place_id BIGINT
-);
-
-drop type nearfeature cascade;
-create type nearfeature as (
-  place_id BIGINT,
-  keywords int[],
-  rank_address integer,
-  rank_search integer,
-  distance float,
-  isguess boolean
-);
-
-drop type nearfeaturecentr cascade;
-create type nearfeaturecentr as (
-  place_id BIGINT,
-  keywords int[],
-  rank_address integer,
-  rank_search integer,
-  distance float,
-  isguess boolean,
-  centroid GEOMETRY
-);
-
-CREATE TABLE location_area_country () INHERITS (location_area_large);
-CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry);
-
-CREATE TABLE search_name_country () INHERITS (search_name_blank);
-CREATE INDEX idx_search_name_country_place_id ON search_name_country USING BTREE (place_id);
-CREATE INDEX idx_search_name_country_name_vector ON search_name_country USING GIN (name_vector) WITH (fastupdate = off);
-
--- start
-CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large);
-CREATE INDEX idx_location_area_large_-partition-_place_id ON location_area_large_-partition- USING BTREE (place_id);
-CREATE INDEX idx_location_area_large_-partition-_geometry ON location_area_large_-partition- USING GIST (geometry);
-
-CREATE TABLE search_name_-partition- () INHERITS (search_name_blank);
-CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id);
-CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid);
-CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off);
-
-CREATE TABLE location_property_-partition- () INHERITS (location_property);
-CREATE INDEX idx_location_property_-partition-_place_id ON location_property_-partition- USING BTREE (place_id);
-CREATE INDEX idx_location_property_-partition-_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id);
-CREATE INDEX idx_location_property_-partition-_housenumber_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id, housenumber);
-
-CREATE TABLE location_road_-partition- (
-  partition integer,
-  place_id BIGINT,
-  country_code VARCHAR(2)
-  );
-SELECT AddGeometryColumn('location_road_-partition-', 'geometry', 4326, 'GEOMETRY', 2);
-CREATE INDEX idx_location_road_-partition-_geometry ON location_road_-partition- USING GIST (geometry);
-CREATE INDEX idx_location_road_-partition-_place_id ON location_road_-partition- USING BTREE (place_id);
-
--- end
-
 create or replace function getNearFeatures(in_partition INTEGER, point GEOMETRY, maxrank INTEGER, isin_tokens INT[]) RETURNS setof nearfeaturecentr AS $$
 DECLARE
   r nearfeaturecentr%rowtype;
@@ -90,10 +31,15 @@ END
 $$
 LANGUAGE plpgsql;
 
-create or replace function deleteLocationArea(in_partition INTEGER, in_place_id BIGINT) RETURNS BOOLEAN AS $$
+create or replace function deleteLocationArea(in_partition INTEGER, in_place_id BIGINT, in_rank_search INTEGER) RETURNS BOOLEAN AS $$
 DECLARE
 BEGIN
 
+  IF in_rank_search <= 4 THEN
+    DELETE from location_area_country WHERE place_id = in_place_id;
+    RETURN TRUE;
+  END IF;
+
 -- start
   IF in_partition = -partition- THEN
     DELETE from location_area_large_-partition- WHERE place_id = in_place_id;
@@ -179,7 +125,7 @@ BEGIN
           FROM search_name_-partition-
           WHERE name_vector @> ARRAY[isin_token]
           AND ST_DWithin(centroid, point, 0.01) 
-          AND search_rank between 22 and 27
+          AND search_rank between 26 and 27
       ORDER BY distance ASC limit 1
     LOOP
       RETURN NEXT r;
@@ -193,6 +139,35 @@ END
 $$
 LANGUAGE plpgsql;
 
+create or replace function getNearestNamedPlaceFeature(in_partition INTEGER, point GEOMETRY, isin_token INTEGER) 
+  RETURNS setof nearfeature AS $$
+DECLARE
+  r nearfeature%rowtype;
+BEGIN
+
+-- start
+  IF in_partition = -partition- THEN
+    FOR r IN 
+      SELECT place_id, name_vector, address_rank, search_rank,
+          ST_Distance(centroid, point) as distance, null as isguess
+          FROM search_name_-partition-
+          WHERE name_vector @> ARRAY[isin_token]
+          AND ST_DWithin(centroid, point, 0.03) 
+          AND search_rank between 16 and 22
+      ORDER BY distance ASC limit 1
+    LOOP
+      RETURN NEXT r;
+    END LOOP;
+    RETURN;
+  END IF;
+-- end
+
+  RAISE EXCEPTION 'Unknown partition %', in_partition;
+END
+$$
+LANGUAGE plpgsql;
+
+
 create or replace function getNearestPostcode(in_partition INTEGER, point GEOMETRY) 
   RETURNS TEXT AS $$
 DECLARE
@@ -220,7 +195,7 @@ create or replace function insertSearchName(
   in_partition INTEGER, in_place_id BIGINT, in_country_code VARCHAR(2), 
   in_name_vector INTEGER[], in_nameaddress_vector INTEGER[],
   in_rank_search INTEGER, in_rank_address INTEGER, in_importance FLOAT,
-  in_centroid GEOMETRY) RETURNS BOOLEAN AS $$
+  in_centroid GEOMETRY, in_geometry GEOMETRY) RETURNS BOOLEAN AS $$
 DECLARE
 BEGIN
 
@@ -230,16 +205,16 @@ BEGIN
 
   IF in_rank_search <= 4 THEN
     DELETE FROM search_name_country WHERE place_id = in_place_id;
-    INSERT INTO search_name_country values (in_place_id, in_rank_search, in_rank_address, in_importance, in_country_code, 
-      in_name_vector, in_nameaddress_vector, in_centroid);
+    INSERT INTO search_name_country values (in_place_id, in_rank_search, in_rank_address, 
+      in_name_vector, in_geometry);
     RETURN TRUE;
   END IF;
 
 -- start
   IF in_partition = -partition- THEN
     DELETE FROM search_name_-partition- values WHERE place_id = in_place_id;
-    INSERT INTO search_name_-partition- values (in_place_id, in_rank_search, in_rank_address, 0, in_country_code, 
-      in_name_vector, in_nameaddress_vector, in_centroid);
+    INSERT INTO search_name_-partition- values (in_place_id, in_rank_search, in_rank_address, 
+      in_name_vector, in_geometry);
     RETURN TRUE;
   END IF;
 -- end

sql/partition-tables.src.sql

@@ -0,0 +1,68 @@
+drop type if exists nearplace cascade;
+create type nearplace as (
+  place_id BIGINT
+);
+
+drop type if exists nearfeature cascade;
+create type nearfeature as (
+  place_id BIGINT,
+  keywords int[],
+  rank_address integer,
+  rank_search integer,
+  distance float,
+  isguess boolean
+);
+
+drop type if exists nearfeaturecentr cascade;
+create type nearfeaturecentr as (
+  place_id BIGINT,
+  keywords int[],
+  rank_address integer,
+  rank_search integer,
+  distance float,
+  isguess boolean,
+  centroid GEOMETRY
+);
+
+drop table IF EXISTS search_name_blank CASCADE;
+CREATE TABLE search_name_blank (
+  place_id BIGINT,
+  search_rank integer,
+  address_rank integer,
+  name_vector integer[]
+  );
+SELECT AddGeometryColumn('search_name_blank', 'centroid', 4326, 'GEOMETRY', 2);
+
+
+CREATE TABLE location_area_country () INHERITS (location_area_large);
+CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry);
+
+CREATE TABLE search_name_country () INHERITS (search_name_blank);
+CREATE INDEX idx_search_name_country_place_id ON search_name_country USING BTREE (place_id);
+CREATE INDEX idx_search_name_country_name_vector ON search_name_country USING GIN (name_vector) WITH (fastupdate = off);
+
+-- start
+CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large);
+CREATE INDEX idx_location_area_large_-partition-_place_id ON location_area_large_-partition- USING BTREE (place_id);
+CREATE INDEX idx_location_area_large_-partition-_geometry ON location_area_large_-partition- USING GIST (geometry);
+
+CREATE TABLE search_name_-partition- () INHERITS (search_name_blank);
+CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id);
+CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid);
+CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off);
+
+CREATE TABLE location_property_-partition- () INHERITS (location_property);
+CREATE INDEX idx_location_property_-partition-_place_id ON location_property_-partition- USING BTREE (place_id);
+CREATE INDEX idx_location_property_-partition-_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id);
+CREATE INDEX idx_location_property_-partition-_housenumber_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id, housenumber);
+
+CREATE TABLE location_road_-partition- (
+  partition integer,
+  place_id BIGINT,
+  country_code VARCHAR(2)
+  );
+SELECT AddGeometryColumn('location_road_-partition-', 'geometry', 4326, 'GEOMETRY', 2);
+CREATE INDEX idx_location_road_-partition-_geometry ON location_road_-partition- USING GIST (geometry);
+CREATE INDEX idx_location_road_-partition-_place_id ON location_road_-partition- USING BTREE (place_id);
+
+-- end

sql/tables.sql

@@ -32,6 +32,7 @@ CREATE TABLE query_log (
   results integer
   );
 CREATE INDEX idx_query_log ON query_log USING BTREE (starttime);
+GRANT SELECT ON query_log TO "www-data" ;
 GRANT INSERT ON query_log TO "www-data" ;
 GRANT UPDATE ON query_log TO "www-data" ;
 
@@ -69,7 +70,6 @@ drop table IF EXISTS word;
 CREATE TABLE word (
   word_id INTEGER,
   word_token text,
-  word_trigram text,
   word text,
   class text,
   type text,
@@ -77,9 +77,7 @@ CREATE TABLE word (
   search_name_count INTEGER,
   operator TEXT
   );
-SELECT AddGeometryColumn('word', 'location', 4326, 'GEOMETRY', 2);
 CREATE INDEX idx_word_word_token on word USING BTREE (word_token);
---CREATE INDEX idx_word_trigram ON word USING gin(word_trigram gin_trgm_ops) WITH (fastupdate = off);
 GRANT SELECT ON word TO "www-data" ;
 DROP SEQUENCE seq_word;
 CREATE SEQUENCE seq_word start 1;
@@ -123,8 +121,8 @@ CREATE INDEX idx_location_property_tiger_parent_place_id ON location_property_ti
 CREATE INDEX idx_location_property_tiger_housenumber_parent_place_id ON location_property_tiger USING BTREE (parent_place_id, housenumber);
 GRANT SELECT ON location_property_tiger TO "www-data";
 
-drop table IF EXISTS search_name_blank CASCADE;
-CREATE TABLE search_name_blank (
+drop table IF EXISTS search_name;
+CREATE TABLE search_name (
   place_id BIGINT,
   search_rank integer,
   address_rank integer,
@@ -133,10 +131,7 @@ CREATE TABLE search_name_blank (
   name_vector integer[],
   nameaddress_vector integer[]
   );
-SELECT AddGeometryColumn('search_name_blank', 'centroid', 4326, 'GEOMETRY', 2);
-
-drop table IF EXISTS search_name;
-CREATE TABLE search_name () INHERITS (search_name_blank);
+SELECT AddGeometryColumn('search_name', 'centroid', 4326, 'GEOMETRY', 2);
 CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id);
 
 drop table IF EXISTS place_addressline;
@@ -248,17 +243,7 @@ CREATE TRIGGER place_before_delete BEFORE DELETE ON place
 CREATE TRIGGER place_before_insert BEFORE INSERT ON place
     FOR EACH ROW EXECUTE PROCEDURE place_insert();
 
-alter table placex add column geometry_sector INTEGER;
-alter table placex add column indexed_status INTEGER;
-alter table placex add column indexed_date TIMESTAMP;
-
-update placex set geometry_sector = geometry_sector(geometry);
-drop index idx_placex_pendingbylatlon;
-drop index idx_placex_interpolation;
 drop index idx_placex_sector;
-CREATE INDEX idx_placex_pendingbylatlon ON placex USING BTREE (geometry_index(geometry_sector,indexed,name),rank_search) 
-  where geometry_index(geometry_sector,indexed,name) IS NOT NULL;
-CREATE INDEX idx_placex_interpolation ON placex USING BTREE (geometry_sector) where indexed = false and class='place' and type='houses';
 CREATE INDEX idx_placex_sector ON placex USING BTREE (geometry_sector,rank_address,osm_type,osm_id);
 
 DROP SEQUENCE seq_postcodes;
@@ -278,6 +263,7 @@ CREATE TABLE import_polygon_error (
 SELECT AddGeometryColumn('import_polygon_error', 'prevgeometry', 4326, 'GEOMETRY', 2);
 SELECT AddGeometryColumn('import_polygon_error', 'newgeometry', 4326, 'GEOMETRY', 2);
 CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id);
+GRANT SELECT ON import_polygon_error TO "www-data";
 
 drop table import_polygon_delete;
 CREATE TABLE import_polygon_delete (
@@ -287,6 +273,7 @@ CREATE TABLE import_polygon_delete (
   type TEXT NOT NULL
   );
 CREATE INDEX idx_import_polygon_delete_osmid ON import_polygon_delete USING BTREE (osm_type, osm_id);
+GRANT SELECT ON import_polygon_delete TO "www-data";
 
 drop sequence file;
 CREATE SEQUENCE file start 1;

utils/blocks.php

@@ -0,0 +1,52 @@
+#!/usr/bin/php -Cq
+<?php
+
+        require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
+        ini_set('memory_limit', '800M');
+
+	$aCMDOptions = array(
+		"Manage service blocks / restrictions",
+		array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+		array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+		array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+		array('list', 'l', 0, 1, 0, 0, 'bool', 'List recent blocks'),
+		array('delete', 'd', 0, 1, 0, 0, 'bool', 'Clear recent blocks list'),
+		array('flush', '', 0, 1, 0, 0, 'bool', 'Flush all blocks / stats'),
+	);
+	getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
+
+	$m = getBucketMemcache();
+        if (!$m)
+	{
+		echo "ERROR: Bucket memcache is not configured\n";
+		exit;
+	}
+
+	if ($aResult['list'])
+	{
+		$iCurrentSleeping = $m->get('sleepCounter');
+		echo "\n Sleeping blocks count: $iCurrentSleeping\n";
+
+		$aBlocks = getBucketBlocks();
+		echo "\n";
+		printf(" %-40s | %12s | %7s | %13s | %31s | %8s\n", "Key", "Total Blocks", "Current", "Still Blocked", "Last Block Time", "Sleeping");
+		printf(" %'--40s-|-%'-12s-|-%'-7s-|-%'-13s-|-%'-31s-|-%'-8s\n", "", "", "", "", "", "");
+		foreach($aBlocks as $sKey => $aDetails)
+		{
+			printf(" %-40s | %12s | %7s | %13s | %31s | %8s\n", $sKey, $aDetails['totalBlocks'], 
+				(int)$aDetails['currentBucketSize'], $aDetails['currentlyBlocked']?'Y':'N', 
+				date("r", $aDetails['lastBlockTimestamp']), $aDetails['isSleeping']?'Y':'N');
+		}
+		echo "\n";
+	}
+
+	if ($aResult['delete'])
+	{
+		$m->set('sleepCounter', 0);
+		clearBucketBlocks();
+	}
+
+	if ($aResult['flush'])
+	{
+		$m->flush();
+	}

utils/country_languages.php

@@ -0,0 +1,34 @@
+#!/usr/bin/php -Cq
+<?php
+
+        require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
+        ini_set('memory_limit', '800M');
+        ini_set('display_errors', 'stderr');
+
+        $aCMDOptions = array(
+                "Import country language data from osm wiki",
+                array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+                array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+                array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+        );
+        getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
+
+	include(CONST_BasePath.'/settings/phrase_settings.php');
+
+	if (true)
+	{
+		$sURL = 'http://wiki.openstreetmap.org/wiki/Special:Export/Nominatim/Country_Codes';
+		$sWikiPageXML = file_get_contents($sURL);
+		if (preg_match_all('#\\| ([a-z]{2}) \\|\\| [^|]+\\|\\| ([a-z,]+)#', $sWikiPageXML, $aMatches, PREG_SET_ORDER))
+		{
+			foreach($aMatches as $aMatch)
+			{
+				$aLanguages = explode(',', $aMatch[2]);
+				foreach($aLanguages as $i => $s)
+				{
+					$aLanguages[$i] = '"'.pg_escape_string($s).'"';
+				}
+				echo "UPDATE country_name set country_default_language_codes = '{".join(',',$aLanguages)."}' where country_code = '".pg_escape_string($aMatch[1])."';\n";
+			}
+		}
+	}

utils/importWikipedia.php

@@ -0,0 +1,593 @@
+#!/usr/bin/php -Cq
+<?php
+
+	require_once(dirname(dirname(__FILE__)).'/lib/init-cmd.php');
+	ini_set('memory_limit', '800M');
+
+	$aCMDOptions = array(
+		"Create and setup nominatim search system",
+		array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
+		array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
+		array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
+
+		array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create wikipedia tables'),
+		array('parse-articles', '', 0, 1, 0, 0, 'bool', 'Parse wikipedia articles'),
+		array('link', '', 0, 1, 0, 0, 'bool', 'Try to link to existing OSM ids'),
+	);
+	getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
+
+/*
+$sTestPageText = <<<EOD
+{{Coord|47|N|2|E|type:country_region:FR|display=title}}
+{{ Infobox Amusement park
+| name = Six Flags Great Adventure
+| image = [[File:SixFlagsGreatAdventure logo.png]]
+| caption = Six Flags Great Adventure logo
+| location = [[Jackson, New Jersey|Jackson]]
+| location2 = New Jersey
+| location3 = United States
+| address = 1 Six Flags Boulevard<ref name="drivedir"/>
+| season = March/April through October/November
+| opening_date = July 1, 1974
+| previous_names = Great Adventure
+| area_acre = 2200
+| rides = 45 park admission rides
+| coasters = 12
+| water_rides = 2
+| owner = [[Six Flags]]
+| general_manager = 
+| homepage = [http://www.sixflags.com/parks/greatadventure/ Six Flags Great Adventure]
+}}
+EOD;
+var_dump(_templatesToProperties(_parseWikipediaContent($sTestPageText)));
+exit;
+//| coordinates = {{Coord|40|08|16.65|N|74|26|26.69|W|region:US-NJ_type:landmark|display=inline,title}}
+*/
+/*
+
+	$a = array();
+	$a[] = 'test';
+
+	$oDB &= getDB();
+
+	if ($aCMDResult['drop-tables'])
+	{
+		$oDB->query('DROP TABLE wikipedia_article');
+		$oDB->query('DROP TABLE wikipedia_link');
+	}
+*/
+
+	if ($aCMDResult['create-tables'])
+	{
+		$sSQL = <<<'EOD'
+CREATE TABLE wikipedia_article (
+    language text NOT NULL,
+    title text NOT NULL,
+    langcount integer,
+    othercount integer,
+    totalcount integer,
+    lat double precision,
+    lon double precision,
+    importance double precision,
+    title_en text,
+    osm_type character(1),
+    osm_id bigint,
+    infobox_type text,
+    population bigint,
+    website text
+);
+		$oDB->query($sSQL);
+
+		$oDB->query("SELECT AddGeometryColumn('wikipedia_article', 'location', 4326, 'GEOMETRY', 2)");
+
+		$sSQL = <<<'EOD'
+CREATE TABLE wikipedia_link (
+  from_id INTEGER,
+  to_name text
+  );
+EOD;
+		$oDB->query($sSQL);
+	}
+
+	function degreesAndMinutesToDecimal($iDegrees, $iMinutes=0, $fSeconds=0, $sNSEW='N')
+	{
+		$sNSEW = strtoupper($sNSEW);
+		return ($sNSEW == 'S' || $sNSEW == 'W'?-1:1) * ((float)$iDegrees + (float)$iMinutes/60 + (float)$fSeconds/3600);
+	}
+
+	function _parseWikipediaContent($sPageText)
+	{
+		$sPageText = str_replace("\n", ' ', $sPageText);
+		$sPageText = preg_replace('#<!--.*?-->#m', '', $sPageText);
+		$sPageText = preg_replace('#<math>.*?<\\/math>#m', '', $sPageText);
+
+		$aPageText = preg_split('#({{|}}|\\[\\[|\\]\\]|[|])#', $sPageText, -1, PREG_SPLIT_DELIM_CAPTURE);
+
+		$aPageProperties = array();
+		$sPageBody = '';
+		$aTemplates = array();
+		$aLinks = array();
+
+		$aTemplateStack = array();
+		$aState = array('body');
+		foreach($aPageText as $i => $sPart)
+		{
+			switch($sPart)
+			{
+			case '{{':
+				array_unshift($aTemplateStack, array('', array()));
+				array_unshift($aState, 'template');
+				break;
+			case '}}':
+				if ($aState[0] == 'template' || $aState[0] == 'templateparam')
+				{
+					$aTemplate = array_shift($aTemplateStack);
+					array_shift($aState);
+
+					$aTemplates[] = $aTemplate;
+
+				}
+				break;
+			case '[[':
+				$sLinkPage = '';
+				$sLinkSyn = '';
+				array_unshift($aState, 'link');
+				break;
+			case ']]':
+				if ($aState[0] == 'link' || $aState[0] == 'linksynonim')
+				{
+					if (!$sLinkSyn) $sLinkSyn = $sLinkPage;
+					if (substr($sLinkPage, 0, 6) == 'Image:') $sLinkSyn = substr($sLinkPage, 6);
+
+					$aLinks[] = array($sLinkPage, $sLinkSyn);
+
+					array_shift($aState);
+					switch($aState[0])
+					{
+					case 'template':
+						$aTemplateStack[0][0] .= trim($sPart);
+						break;
+					case 'templateparam':
+						$aTemplateStack[0][1][0] .= $sLinkSyn;
+						break;
+					case 'link':
+						$sLinkPage .= trim($sPart);
+						break;
+					case 'linksynonim':
+						$sLinkSyn .= $sPart;
+						break;
+					case 'body':
+						$sPageBody .= $sLinkSyn;
+						break;
+					default:
+						var_dump($aState, $sPageName, $aTemplateStack, $sPart, $aPageText);
+						fail('unknown state');
+					}
+				}
+				break;
+			case '|':
+				if ($aState[0] == 'template' || $aState[0] == 'templateparam')
+				{
+					// Create a new template paramater
+					$aState[0] = 'templateparam';
+					array_unshift($aTemplateStack[0][1], '');
+				}
+				if ($aState[0] == 'link') $aState[0] = 'linksynonim';
+				break;
+			default:
+				switch($aState[0])
+				{
+				case 'template':
+					$aTemplateStack[0][0] .= trim($sPart);
+					break;
+				case 'templateparam':
+					$aTemplateStack[0][1][0] .= $sPart;
+					break;
+				case 'link':
+					$sLinkPage .= trim($sPart);
+					break;
+				case 'linksynonim':
+					$sLinkSyn .= $sPart;
+					break;
+				case 'body':
+					$sPageBody .= $sPart;
+					break;
+				default:
+					var_dump($aState, $aPageText);
+					fail('unknown state');
+				}
+				break;
+			}
+		}
+		return $aTemplates;
+	}
+
+	function _templatesToProperties($aTemplates)
+	{
+		$aPageProperties = array();
+		foreach($aTemplates as $iTemplate => $aTemplate)
+		{
+			$aParams = array();
+			foreach(array_reverse($aTemplate[1]) as $iParam => $sParam)
+			{
+				if (($iPos = strpos($sParam, '=')) === FALSE)
+				{
+					$aParams[] = trim($sParam);
+				}
+				else
+				{
+					$aParams[trim(substr($sParam, 0, $iPos))] = trim(substr($sParam, $iPos+1));
+				}
+			}
+			$aTemplates[$iTemplate][1] = $aParams;
+			if (!isset($aPageProperties['sOfficialName']) && isset($aParams['official_name']) && $aParams['official_name']) $aPageProperties['sOfficialName'] = $aParams['official_name'];
+			if (!isset($aPageProperties['iPopulation']) && isset($aParams['population']) && $aParams['population'] && preg_match('#^[0-9.,]+#', $aParams['population']))
+			{
+				$aPageProperties['iPopulation'] = (int)str_replace(array(',','.'), '', $aParams['population']);
+			}
+			if (!isset($aPageProperties['iPopulation']) && isset($aParams['population_total']) && $aParams['population_total'] && preg_match('#^[0-9.,]+#', $aParams['population_total']))
+			{
+				$aPageProperties['iPopulation'] = (int)str_replace(array(',','.'), '', $aParams['population_total']);
+			}
+			if (!isset($aPageProperties['iPopulation']) && isset($aParams['population_urban']) && $aParams['population_urban'] && preg_match('#^[0-9.,]+#', $aParams['population_urban']))
+			{
+				$aPageProperties['iPopulation'] = (int)str_replace(array(',','.'), '', $aParams['population_urban']);
+			}
+			if (!isset($aPageProperties['iPopulation']) && isset($aParams['population_estimate']) && $aParams['population_estimate'] && preg_match('#^[0-9.,]+#', $aParams['population_estimate']))
+			{
+				$aPageProperties['iPopulation'] = (int)str_replace(array(',','.'), '', $aParams['population_estimate']);
+			}
+			if (!isset($aPageProperties['sWebsite']) && isset($aParams['website']) && $aParams['website'])
+			{
+				if (preg_match('#^\\[?([^ \\]]+)[^\\]]*\\]?$#', $aParams['website'], $aMatch))
+				{
+					$aPageProperties['sWebsite'] = $aMatch[1];
+					if (strpos($aPageProperties['sWebsite'],':/'.'/') === FALSE)
+					{
+						$aPageProperties['sWebsite'] = 'http:/'.'/'.$aPageProperties['sWebsite'];
+					}
+				}
+			}
+			if (!isset($aPageProperties['sTopLevelDomain']) && isset($aParams['cctld']) && $aParams['cctld'])
+			{
+				$aPageProperties['sTopLevelDomain'] = str_replace(array('[',']','.'),'', $aParams['cctld']);
+			}
+
+			if (!isset($aPageProperties['sInfoboxType']) && strtolower(substr($aTemplate[0],0,7)) == 'infobox')
+			{
+				$aPageProperties['sInfoboxType'] = trim(substr($aTemplate[0],8));
+				// $aPageProperties['aInfoboxParams'] = $aParams;
+			}
+
+			// Assume the first template with lots of params is the type (fallback for infobox)
+			if (!isset($aPageProperties['sPossibleInfoboxType']) && sizeof($aParams) > 10)
+			{
+				$aPageProperties['sPossibleInfoboxType'] = trim($aTemplate[0]);
+				// $aPageProperties['aInfoboxParams'] = $aParams;
+			}
+
+			// do we have a lat/lon
+			if (!isset($aPageProperties['fLat']))
+			{
+				if (isset($aParams['latd']) && isset($aParams['longd']))
+				{
+					$aPageProperties['fLat'] = degreesAndMinutesToDecimal($aParams['latd'], @$aParams['latm'], @$aParams['lats'], @$aParams['latNS']);
+					$aPageProperties['fLon'] = degreesAndMinutesToDecimal($aParams['longd'], @$aParams['longm'], @$aParams['longs'], @$aParams['longEW']);
+				}
+				if (isset($aParams['lat_degrees']) && isset($aParams['lat_degrees']))
+				{
+					$aPageProperties['fLat'] = degreesAndMinutesToDecimal($aParams['lat_degrees'], @$aParams['lat_minutes'], @$aParams['lat_seconds'], @$aParams['lat_direction']);
+					$aPageProperties['fLon'] = degreesAndMinutesToDecimal($aParams['long_degrees'], @$aParams['long_minutes'], @$aParams['long_seconds'], @$aParams['long_direction']);
+				}
+				if (isset($aParams['latitude']) && isset($aParams['longitude']))
+				{
+					if (preg_match('#[0-9.]+#', $aParams['latitude']) && preg_match('#[0-9.]+#', $aParams['longitude']))
+					{
+						$aPageProperties['fLat'] = (float)$aParams['latitude'];
+						$aPageProperties['fLon'] = (float)$aParams['longitude'];
+					}
+				}
+				if (strtolower($aTemplate[0]) == 'coord')
+				{
+					if (isset($aParams[3]) && (strtoupper($aParams[3]) == 'N' || strtoupper($aParams[3]) == 'S'))
+					{
+						$aPageProperties['fLat'] = degreesAndMinutesToDecimal($aParams[0], $aParams[1], $aParams[2], $aParams[3]);
+						$aPageProperties['fLon'] = degreesAndMinutesToDecimal($aParams[4], $aParams[5], $aParams[6], $aParams[7]);
+					}
+					elseif (isset($aParams[0]) && isset($aParams[1]) && isset($aParams[2]) && (strtoupper($aParams[2]) == 'N' || strtoupper($aParams[2]) == 'S'))
+					{
+						$aPageProperties['fLat'] = degreesAndMinutesToDecimal($aParams[0], $aParams[1], 0, $aParams[2]);
+						$aPageProperties['fLon'] = degreesAndMinutesToDecimal($aParams[3], $aParams[4], 0, $aParams[5]);
+					}
+					else if (isset($aParams[0]) && isset($aParams[1]) && (strtoupper($aParams[1]) == 'N' || strtoupper($aParams[1]) == 'S'))
+					{
+						$aPageProperties['fLat'] = (strtoupper($aParams[1]) == 'N'?1:-1) * (float)$aParams[0];
+						$aPageProperties['fLon'] = (strtoupper($aParams[3]) == 'E'?1:-1) * (float)$aParams[2];
+					}
+					else if (isset($aParams[0]) && is_numeric($aParams[0]) && isset($aParams[1]) && is_numeric($aParams[1]))
+					{
+						$aPageProperties['fLat'] = (float)$aParams[0];
+						$aPageProperties['fLon'] = (float)$aParams[1];
+					}
+				}
+				if (isset($aParams['Latitude']) && isset($aParams['Longitude']))
+				{
+					$aParams['Latitude'] = str_replace('&nbsp;',' ',$aParams['Latitude']);
+					$aParams['Longitude'] = str_replace('&nbsp;',' ',$aParams['Longitude']);
+					if (preg_match('#^([0-9]+)°( ([0-9]+)′)? ([NS]) to ([0-9]+)°( ([0-9]+)′)? ([NS])#', $aParams['Latitude'], $aMatch))
+					{
+						$aPageProperties['fLat'] =
+							(degreesAndMinutesToDecimal($aMatch[1], $aMatch[3], 0, $aMatch[4])
+							+degreesAndMinutesToDecimal($aMatch[5], $aMatch[7], 0, $aMatch[8])) / 2;
+					}
+					else if (preg_match('#^([0-9]+)°( ([0-9]+)′)? ([NS])#', $aParams['Latitude'], $aMatch))
+					{
+						$aPageProperties['fLat'] = degreesAndMinutesToDecimal($aMatch[1], $aMatch[3], 0, $aMatch[4]);
+					}
+
+					if (preg_match('#^([0-9]+)°( ([0-9]+)′)? ([EW]) to ([0-9]+)°( ([0-9]+)′)? ([EW])#', $aParams['Longitude'], $aMatch))
+					{
+						$aPageProperties['fLon'] =
+							(degreesAndMinutesToDecimal($aMatch[1], $aMatch[3], 0, $aMatch[4])
+							+degreesAndMinutesToDecimal($aMatch[5], $aMatch[7], 0, $aMatch[8])) / 2;
+					}
+					else if (preg_match('#^([0-9]+)°( ([0-9]+)′)? ([EW])#', $aParams['Longitude'], $aMatch))
+					{
+						$aPageProperties['fLon'] = degreesAndMinutesToDecimal($aMatch[1], $aMatch[3], 0, $aMatch[4]);
+					}
+				}
+			}
+		}
+		if (isset($aPageProperties['sPossibleInfoboxType']))
+		{
+			if (!isset($aPageProperties['sInfoboxType'])) $aPageProperties['sInfoboxType'] = '#'.$aPageProperties['sPossibleInfoboxType'];
+			unset($aPageProperties['sPossibleInfoboxType']);
+		}
+		return $aPageProperties;
+	}
+
+	if (isset($aCMDResult['parse-wikipedia']))
+	{
+		$oDB =& getDB();
+		$aArticleNames = $oDB->getCol('select page_title from content where page_namespace = 0 and page_id %10 = '.$aCMDResult['parse-wikipedia'].' and (page_content ilike \'%{{Coord%\' or (page_content ilike \'%lat%\' and page_content ilike \'%lon%\'))');
+//		$aArticleNames = $oDB->getCol($sSQL = 'select page_title from content where page_namespace = 0 and (page_content ilike \'%{{Coord%\' or (page_content ilike \'%lat%\' and page_content ilike \'%lon%\')) and page_title in (\'Virginia\')');
+		foreach($aArticleNames as $sArticleName)
+		{
+			$sPageText = $oDB->getOne('select page_content from content where page_namespace = 0 and page_title = \''.pg_escape_string($sArticleName).'\'');
+			$aP = _templatesToProperties(_parseWikipediaContent($sPageText));
+
+			if (isset($aP['sInfoboxType']))
+			{
+				$aP['sInfoboxType'] = preg_replace('#\\s+#',' ',$aP['sInfoboxType']);
+				$sSQL = 'update wikipedia_article set ';
+				$sSQL .= 'infobox_type = \''.pg_escape_string($aP['sInfoboxType']).'\'';
+				$sSQL .= ' where language = \'en\' and title = \''.pg_escape_string($sArticleName).'\';';
+				$oDB->query($sSQL);
+			}
+			if (isset($aP['iPopulation']))
+			{
+				$sSQL = 'update wikipedia_article set ';
+				$sSQL .= 'population = \''.pg_escape_string($aP['iPopulation']).'\'';
+				$sSQL .= ' where language = \'en\' and title = \''.pg_escape_string($sArticleName).'\';';
+				$oDB->query($sSQL);
+			}
+			if (isset($aP['sWebsite']))
+			{
+				$sSQL = 'update wikipedia_article set ';
+				$sSQL .= 'website = \''.pg_escape_string($aP['sWebsite']).'\'';
+				$sSQL .= ' where language = \'en\' and title = \''.pg_escape_string($sArticleName).'\';';
+				$oDB->query($sSQL);
+			}
+			if (isset($aP['fLat']) && ($aP['fLat']!='-0' || $aP['fLon']!='-0'))
+			{
+				if (!isset($aP['sInfoboxType'])) $aP['sInfoboxType'] = '';
+				echo $sArticleName.'|'.$aP['sInfoboxType'].'|'.$aP['fLat'].'|'.$aP['fLon'] ."\n";
+				$sSQL = 'update wikipedia_article set ';
+				$sSQL .= 'lat = \''.pg_escape_string($aP['fLat']).'\',';
+				$sSQL .= 'lon = \''.pg_escape_string($aP['fLon']).'\'';
+				$sSQL .= ' where language = \'en\' and title = \''.pg_escape_string($sArticleName).'\';';
+				$oDB->query($sSQL);
+			}
+		}
+	}
+
+	function nominatimXMLStart($hParser, $sName, $aAttr)
+	{
+        	global $aNominatRecords;
+	        switch($sName)
+        	{
+	        case 'PLACE':
+        	        $aNominatRecords[] = $aAttr;
+                	break;
+	        }
+	}
+
+	function nominatimXMLEnd($hParser, $sName)
+	{
+	}
+
+
+	if (isset($aCMDResult['link']))
+	{
+		$oDB =& getDB();
+		$aWikiArticles = $oDB->getAll("select * from wikipedia_article where language = 'en' and lat is not null and osm_type is null and totalcount < 31 order by importance desc limit 200000");
+
+		// If you point this script at production OSM you will be blocked
+		$sNominatimBaseURL = 'http://SEVERNAME/search.php';
+
+		foreach($aWikiArticles as $aRecord)
+		{
+			$aRecord['name'] = str_replace('_',' ',$aRecord['title']);
+
+			$sURL = $sNominatimBaseURL.'?format=xml&accept-language=en';
+
+			echo "\n-- ".$aRecord['name'].", ".$aRecord['infobox_type']."\n";
+			$fMaxDist = 0.0000001;
+			$bUnknown = false;
+			switch(strtolower($aRecord['infobox_type']))
+			{
+			case 'former country':
+				continue 2;
+			case 'sea':
+				$fMaxDist = 60; // effectively turn it off
+				$sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
+				break;
+			case 'country':
+			case 'island':
+			case 'islands':
+			case 'continent':
+				$fMaxDist = 60; // effectively turn it off
+				$sURL .= "&featuretype=country";
+				$sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
+				break;
+			case 'prefecture japan':
+				$aRecord['name'] = trim(str_replace(' Prefecture',' ', $aRecord['name']));
+			case 'state':
+			case '#us state':
+			case 'county':
+			case 'u.s. state':
+			case 'u.s. state symbols':
+			case 'german state':
+			case 'province or territory of canada';
+			case 'indian jurisdiction';
+			case 'province';
+			case 'french region':
+			case 'region of italy':
+			case 'kommune':
+			case '#australia state or territory':
+			case 'russian federal subject':
+				$fMaxDist = 4;
+				$sURL .= "&featuretype=state";
+				$sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
+				break;
+			case 'protected area':
+				$fMaxDist = 1;
+				$sURL .= "&nearlat=".$aRecord['lat'];
+				$sURL .= "&nearlon=".$aRecord['lon'];
+				$sURL .= "&viewbox=".($aRecord['lon']-$fMaxDist).",".($aRecord['lat']+$fMaxDist).",".($aRecord['lon']+$fMaxDist).",".($aRecord['lat']-$fMaxDist);
+				break;
+			case 'settlement':
+				$bUnknown = true;
+			case 'french commune':
+			case 'italian comune':
+			case 'uk place':
+			case 'italian comune':
+			case 'australian place':
+			case 'german place':
+			case '#geobox':
+			case 'u.s. county':
+			case 'municipality':
+			case 'city japan':
+			case 'russian inhabited locality':
+			case 'finnish municipality/land area':
+			case 'england county':
+			case 'israel municipality':
+			case 'russian city':
+			case 'city':
+				$fMaxDist = 0.2;
+				$sURL .= "&featuretype=settlement";
+				$sURL .= "&viewbox=".($aRecord['lon']-0.5).",".($aRecord['lat']+0.5).",".($aRecord['lon']+0.5).",".($aRecord['lat']-0.5);
+				break;
+			case 'mountain':
+			case 'mountain pass':
+			case 'river':
+			case 'lake':
+			case 'airport':
+				$fMaxDist = 0.2;
+				$sURL .= "&viewbox=".($aRecord['lon']-0.5).",".($aRecord['lat']+0.5).",".($aRecord['lon']+0.5).",".($aRecord['lat']-0.5);
+
+			case 'ship begin':
+				$fMaxDist = 0.1;
+				$aTypes = array('wreck');
+				$sURL .= "&viewbox=".($aRecord['lon']-0.01).",".($aRecord['lat']+0.01).",".($aRecord['lon']+0.01).",".($aRecord['lat']-0.01);
+				$sURL .= "&nearlat=".$aRecord['lat'];
+				$sURL .= "&nearlon=".$aRecord['lon'];
+				break;
+			case 'road':
+			case 'university':
+			case 'company':
+			case 'department':
+				$fMaxDist = 0.005;
+				$sURL .= "&viewbox=".($aRecord['lon']-0.01).",".($aRecord['lat']+0.01).",".($aRecord['lon']+0.01).",".($aRecord['lat']-0.01);
+				$sURL .= "&bounded=1";
+				$sURL .= "&nearlat=".$aRecord['lat'];
+				$sURL .= "&nearlon=".$aRecord['lon'];
+				break;
+			default:
+				$bUnknown = true;
+				$fMaxDist = 0.005;
+				$sURL .= "&viewbox=".($aRecord['lon']-0.01).",".($aRecord['lat']+0.01).",".($aRecord['lon']+0.01).",".($aRecord['lat']-0.01);
+//				$sURL .= "&bounded=1";
+				$sURL .= "&nearlat=".$aRecord['lat'];
+				$sURL .= "&nearlon=".$aRecord['lon'];
+				echo "-- Unknown: ".$aRecord['infobox_type']."\n";
+				break;
+			}
+			$sNameURL = $sURL.'&q='.urlencode($aRecord['name']);
+
+			var_Dump($sNameURL);
+			$sXML = file_get_contents($sNameURL);
+
+			$aNominatRecords = array();
+			$hXMLParser = xml_parser_create();
+			xml_set_element_handler($hXMLParser, 'nominatimXMLStart', 'nominatimXMLEnd');
+			xml_parse($hXMLParser, $sXML, true);
+			xml_parser_free($hXMLParser);
+
+			if (!isset($aNominatRecords[0]))
+			{
+				$aNameParts = preg_split('#[(,]#',$aRecord['name']);
+				if (sizeof($aNameParts) > 1)
+				{
+					$sNameURL = $sURL.'&q='.urlencode(trim($aNameParts[0]));
+					var_Dump($sNameURL);
+					$sXML = file_get_contents($sNameURL);
+
+					$aNominatRecords = array();
+					$hXMLParser = xml_parser_create();
+					xml_set_element_handler($hXMLParser, 'nominatimXMLStart', 'nominatimXMLEnd');
+					xml_parse($hXMLParser, $sXML, true);
+					xml_parser_free($hXMLParser);#
+				}
+			}
+
+			// assume first is best/right
+			for($i = 0; $i < sizeof($aNominatRecords); $i++)
+			{
+				$fDiff = ($aRecord['lat']-$aNominatRecords[$i]['LAT']) * ($aRecord['lat']-$aNominatRecords[$i]['LAT']);
+				$fDiff += ($aRecord['lon']-$aNominatRecords[$i]['LON']) * ($aRecord['lon']-$aNominatRecords[$i]['LON']);
+				$fDiff = sqrt($fDiff);
+				if ($bUnknown) {
+					// If it was an unknown type base it on the rank of the found result
+					$iRank = (int)$aNominatRecords[$i]['PLACE_RANK'];
+					if ($iRank <= 4) $fMaxDist = 2;
+					elseif ($iRank <= 8) $fMaxDist = 1;
+					elseif ($iRank <= 10) $fMaxDist = 0.8;
+					elseif ($iRank <= 12) $fMaxDist = 0.6;
+					elseif ($iRank <= 17) $fMaxDist = 0.2;
+					elseif ($iRank <= 18) $fMaxDist = 0.1;
+					elseif ($iRank <= 22) $fMaxDist = 0.02;
+					elseif ($iRank <= 26) $fMaxDist = 0.001;
+					else $fMaxDist = 0.001;
+				}
+				echo "-- FOUND \"".substr($aNominatRecords[$i]['DISPLAY_NAME'],0,50)."\", ".$aNominatRecords[$i]['CLASS'].", ".$aNominatRecords[$i]['TYPE'].", ".$aNominatRecords[$i]['PLACE_RANK'].", ".$aNominatRecords[$i]['OSM_TYPE']." (dist:$fDiff, max:$fMaxDist)\n";
+				if ($fDiff > $fMaxDist)
+				{
+					echo "-- Diff too big $fDiff (max: $fMaxDist)".$aRecord['lat'].','.$aNominatRecords[$i]['LAT'].' & '.$aRecord['lon'].','.$aNominatRecords[$i]['LON']." \n";
+				}
+				else
+				{
+					$sSQL = "update wikipedia_article set osm_type=";
+					switch($aNominatRecords[$i]['OSM_TYPE'])
+					{
+					case 'relation': $sSQL .= "'R'"; break;
+					case 'way': $sSQL .= "'W'"; break;
+					case 'node': $sSQL .= "'N'"; break;
+					}
+					$sSQL .= ", osm_id=".$aNominatRecords[$i]['OSM_ID']." where language = '".pg_escape_string($aRecord['language'])."' and title = '".pg_escape_string($aRecord['title'])."'";
+					$oDB->query($sSQL);
+					break;
+				}
+			}
+		}
+	}

utils/import_wikipedia.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+psqlcmd=psql wikipedia2013
+mysql2pgsqlcmd=./mysql2pgsql.perl /dev/stdin /dev/stdout
+
+language=( "ar" "bg" "ca" "cs" "da" "de" "en" "es" "eo" "eu" "fa" "fr" "ko" "hi" "hr" "id" "it" "he" "lt" "hu" "ms" "nl" "ja" "no" "pl" "pt" "kk" "ro" "ru" "sk" "sl" "sr" "fi" "sv" "tr" "uk" "vi" "vo" "war" "zh" )
+
+# wikipedia pages and links
+echo "CREATE TABLE linkcounts (language text, title text, count integer, sumcount integer, lat double, lon double );"  | $psqlcmd
+echo "CREATE TABLE wikipedia_redirect (language text, from_title text, to_title text );"  | $psqlcmd
+
+for i in "${language[@]}"
+do
+	wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-page.sql.gz
+	wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-pagelinks.sql.gz
+	wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-langlinks.sql.gz
+	wget http://dumps.wikimedia.org/${i}wiki/latest/${i}wiki-latest-redirect.sql.gz
+done
+
+for i in "${language[@]}"
+do
+  gzip -dc ${i}wiki-latest-pagelinks.sql.gz | sed "s/\`pagelinks\`/\`${i}pagelinks\`/g" | $mysql2pgsqlcmd | $psqlcmd
+  gzip -dc ${i}wiki-latest-page.sql.gz | sed "s/\`page\`/\`${i}page\`/g" | $mysql2pgsqlcmd | $psqlcmd
+  gzip -dc ${i}wiki-latest-langlinks.sql.gz | sed "s/\`langlinks\`/\`${i}langlinks\`/g" | $mysql2pgsqlcmd | $psqlcmd
+  gzip -dc ${i}wiki-latest-redirect.sql.gz | sed "s/\`redirect\`/\`${i}redirect\`/g" | $mysql2pgsqlcmd | $psqlcmd
+done
+
+for i in "${language[@]}"
+do
+  echo "create table ${i}pagelinkcount as select pl_title as title,count(*) as count from ${i}pagelinks where pl_namespace = 0 group by pl_title;" | $psqlcmd
+  echo "insert into linkcounts select '${i}',pl_title,count(*) from ${i}pagelinks where pl_namespace = 0 group by pl_title;" | $psqlcmd
+  echo "insert into wikipedia_redirect select '${i}',page_title,rd_title from ${i}redirect join ${i}page on (rd_from = page_id) where page_namespace = 0 and rd_namespace = 0;" | $psqlcmd
+  echo "alter table ${i}pagelinkcount add column othercount integer;" | $psqlcmd
+  echo "update ${i}pagelinkcount set othercount = 0;" | $psqlcmd
+  for j in "${language[@]}"
+  do
+    echo "update ${i}pagelinkcount set othercount = ${i}pagelinkcount.othercount + x.count from (select page_title as title,count from ${i}langlinks join ${i}page on (ll_from = page_id) join ${j}pagelinkcount on (ll_lang = '${j}' and ll_title = title)) as x where x.title = ${i}pagelinkcount.title;" | $psqlcmd
+  done
+  echo "insert into wikipedia_article select '${i}', title, count, othercount, count+othercount from ${i}pagelinkcount;" | $psqlcmd
+done
+
+echo "update wikipedia_article set importance = log(totalcount)/log((select max(totalcount) from wikipedia_article))" | $psqlcmd
+
+# precalculated lat,lon from dbpedia
+wget http://downloads.dbpedia.org/current/en/geo_coordinates_en.nq.bz2
+bzip2 -dc geo_coordinates_en.nq.bz2 | grep http://www.georss.org/georss/point | sed 's|<http://dbpedia.org/resource/[^>]*> *<http://www.georss.org/georss/point> "\(-\?[-0-9.E]\+\) \(-\?[-0-9.E]\+\)"@en <http://\([a-z][a-z]\).wikipedia.org/wiki/\([^#]\+\)#> .|update pagelinks set lat=\1, lon=\2 where language = '"'"'\3'"'"' and title = decode_url_part('"'"'\4'"'"');|g' | $psqlcmd
+
+# media wiki dumper
+wget https://github.com/bcollier/mwdumper/blob/master/build/mwdumper.jar
+
+# latest english wikipedia articles
+wget http://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2
+java -jar mwdumper.jar --format=sql:1.5 enwiki-latest-pages-articles.xml.bz2 | ./mysql2pgsql.perl /dev/stdin /dev/stdout | sed 's/"text (/text ("/g' | sed 's/"old_flags)"/"old_flags")/g' | sed 's/"revision (/revision ("/g' | sed 's/"rev_deleted)"/"rev_deleted")/g' | sed 's/"page (/page ("/g' | sed 's/"page_len)"/"page_len")/g' | sed "s/DATE_ADD(E'1970-01-01', INTERVAL UNIX_TIMESTAMP() SECOND)[+]//g" | sed 's/RAND()/0/g' | $psqlcmd

utils/imports.php

@@ -11,7 +11,7 @@
 		array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
 
 		array('parse-tiger', '', 0, 1, 1, 1, 'realpath', 'Convert tiger edge files to nominatim sql import'),
-		array('parse-tiger-2011', '', 0, 1, 1, 1, 'realpath', 'Convert tiger edge files to nominatim sql import (source: edges directory of tiger data)'),
+		array('parse-tiger-2011', '', 0, 1, 1, 1, 'realpath', 'Convert tiger edge files to nominatim sql import - datafiles from 2011 or later (source: edges directory of tiger data)'),
 	);
 	getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
 
@@ -98,7 +98,7 @@
 
 
 		$bDidSomething = true;
-		foreach(glob($aCMDResult['parse-tiger-2011'].'/tl_2011_?????_edges.zip', 0) as $sImportFile)
+		foreach(glob($aCMDResult['parse-tiger-2011'].'/tl_20??_?????_edges.zip', 0) as $sImportFile)
 		{
 			set_time_limit(30);
 			preg_match('#([0-9]{5})_(.*)#',basename($sImportFile), $aMatch);

utils/server_compare.php

@@ -0,0 +1,78 @@
+#!/usr/bin/php -Cq
+<?php
+
+	// Apache log file
+	$sFile = "sample.log.txt";
+	$sHost1 = 'http://mq-open-search-lm02.ihost.aol.com:8000/nominatim/v1';
+	$sHost2 = 'http://mq-open-search-lm03.ihost.aol.com:8000/nominatim/v1';
+
+
+	$sHost1Escaped = str_replace('/', '\\/', $sHost1);
+	$sHost2Escaped = str_replace('/', '\\/', $sHost2);
+
+	$aToDo = array(251, 293, 328, 399.1, 455.1, 479, 496, 499, 574, 609, 702, 790, 846, 865, 878, 894, 902, 961, 980);
+
+	$hFile = @fopen($sFile, "r");
+	if (!$hFile)
+	{
+		echo "Unable to open file: $sFile\n";
+		exit;
+	}
+
+	$i = 0;
+	while (($sLine = fgets($hFile, 10000)) !== false)
+	{
+		$i++;
+		if (!in_array($i, $aToDo)) continue;
+
+		if (preg_match('#"GET (.*) HTTP/1.[01]"#', $sLine, $aResult))
+		{
+			$sURL1 = $sHost1.$aResult[1];
+			$sURL2 = $sHost2.$aResult[1];
+
+			$sRes1 = '';
+			$k = 0;
+			while(!$sRes1 && $k < 10)
+			{
+				$sRes1 = file_get_contents($sURL1);
+				$k++;
+				if (!$sRes1) sleep(10);
+			}
+			$sRes2 = file_get_contents($sURL2);
+
+			// Strip out the things that will always change
+			$sRes1 =  preg_replace('# timestamp=\'[^\']*\'#', '', $sRes1);
+			$sRes1 =  str_replace($sHost1, '', $sRes1);
+			$sRes1 =  str_replace($sHost1Escaped, '', $sRes1);
+			$sRes2 =  preg_replace('# timestamp=\'[^\']*\'#', '', $sRes2);
+			$sRes2 =  str_replace($sHost2, '', $sRes2);
+			$sRes2 =  str_replace($sHost2Escaped, '', $sRes2);
+
+			if ($sRes1 != $sRes2)
+			{
+				echo "$i:\n";
+				var_dump($sURL1, $sURL2);
+
+				$sRes = $sURL1.":\n";
+				for ($j = 0; $j < strlen($sRes1); $j+=40)
+				{
+					$sRes  .= substr($sRes1, $j, 40)."\n";
+				}
+				file_put_contents('log/'.$i.'.1', $sRes);
+
+				$sRes = $sURL2.":\n";
+				for ($j = 0; $j < strlen($sRes2); $j+=40)
+				{
+					$sRes  .= substr($sRes2, $j, 40)."\n";
+				}
+				file_put_contents('log/'.$i.'.2', $sRes);
+			}
+			echo ".\n";
+		}
+		else
+		{
+			var_dump($sLine);
+		}
+	}
+
+	fclose($hFile);

utils/setup.php

@@ -24,13 +24,15 @@
 		array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
 		array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
 		array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
-		array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
+		array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
+		array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
+		array('no-partitions', '', 0, 1, 0, 0, 'bool', "Do not partition search indices (speeds up import of single country extracts)"),
 		array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
 		array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
 		array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
 		array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
 		array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
-		array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
+		array('create-roads', '', 0, 1, 0, 0, 'bool', ''),
 		array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
 		array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
 		array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
@@ -95,7 +97,7 @@
 		{
 			fail('database already exists ('.CONST_Database_DSN.')');
 		}
-		passthru('createdb -E UTF-8 '.$aDSNInfo['database']);
+		passthruCheckReturn('createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 	}
 
 	if ($aCMDResult['setup-db'] || $aCMDResult['all'])
@@ -105,14 +107,33 @@
 		// TODO: path detection, detection memory, etc.
 
 		$oDB =& getDB();
-		passthru('createlang plpgsql '.$aDSNInfo['database']);
+
+		$sVersionString = $oDB->getOne('select version()');
+		preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
+		if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
+		{
+			echo "ERROR: PostgreSQL version is not correct.  Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
+			exit;
+		}
+
+		passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
 		$pgver = (float) CONST_Postgresql_Version;
 		if ($pgver < 9.1) {
 			pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
+			pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
 		} else {
 			pgsqlRunScript('CREATE EXTENSION hstore');
 		}
+
 		pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
+		$sVersionString = $oDB->getOne('select postgis_full_version()');
+		preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
+		if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
+		{
+			echo "ERROR: PostGIS version is not correct.  Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
+			exit;
+		}
+
 		pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
@@ -121,7 +142,11 @@
 		pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
-		pgsqlRunScriptFile(CONST_BasePath.'/data/worldboundaries.sql');
+
+		if ($aCMDResult['no-partitions'])
+		{
+			pgsqlRunScript('update country_name set partition = 0');
+		}
 	}
 
 	if ($aCMDResult['import-data'] || $aCMDResult['all'])
@@ -135,8 +160,14 @@
 			echo "Please download and build osm2pgsql.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
 			fail("osm2pgsql not found in '$osm2pgsql'");
 		}
+
+		if (!is_null(CONST_Osm2pgsql_Flatnode_File))
+		{
+			$osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
+		}
 		$osm2pgsql .= ' -lsc -O gazetteer --hstore';
 		$osm2pgsql .= ' -C '.$iCacheMemory;
+		$osm2pgsql .= ' -P '.$aDSNInfo['port'];
 		$osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file'];
 		passthruCheckReturn($osm2pgsql);
 
@@ -202,20 +233,49 @@
 		pgsqlRunScript($sTemplate);
 	}
 
-	if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
+	if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
 	{
-		echo "Partitions\n";
+		echo "Partition Tables\n";
 		$bDidSomething = true;
 		$oDB =& getDB();
-		$sSQL = 'select partition from country_name order by country_code';
+		$sSQL = 'select distinct partition from country_name';
 		$aPartitions = $oDB->getCol($sSQL);
 		if (PEAR::isError($aPartitions))
 		{
 			fail($aPartitions->getMessage());
 		}
-		$aPartitions[] = 0;
+		if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 
-		$sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
+		$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
+		preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
+		foreach($aMatches as $aMatch)
+		{
+			$sResult = '';
+			foreach($aPartitions as $sPartitionName)
+			{
+				$sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
+			}
+			$sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
+		}
+
+		pgsqlRunScript($sTemplate);
+	}
+
+
+	if ($aCMDResult['create-partition-functions'] || $aCMDResult['all'])
+	{
+		echo "Partition Functions\n";
+		$bDidSomething = true;
+		$oDB =& getDB();
+		$sSQL = 'select distinct partition from country_name';
+		$aPartitions = $oDB->getCol($sSQL);
+		if (PEAR::isError($aPartitions))
+		{
+			fail($aPartitions->getMessage());
+		}
+		if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
+
+		$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
 		preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 		foreach($aMatches as $aMatch)
 		{
@@ -283,13 +343,13 @@
 		if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection));
 		echo '.';
 
-		$sSQL = 'select partition from country_name order by country_code';
+		$sSQL = 'select distinct partition from country_name';
 		$aPartitions = $oDB->getCol($sSQL);
 		if (PEAR::isError($aPartitions))
 		{
 			fail($aPartitions->getMessage());
 		}
-		$aPartitions[] = 0;
+		if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 		foreach($aPartitions as $sPartition)
 		{
 			if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection));
@@ -313,7 +373,7 @@
 		{
 			$aDBInstances[$i] =& getDB(true);
 			$sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
-			$sSQL .= 'housenumber, street, isin, postcode, country_code, extratags, ';
+			$sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
 			$sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
 			if ($aCMDResult['verbose']) echo "$sSQL\n";
 			if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
@@ -445,71 +505,83 @@
 		$bDidSomething = true;
 		$oDB =& getDB();
 
-		if (!file_exists(CONST_Osmosis_Binary)) fail("please download osmosis");
-		if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
+		if (!file_exists(CONST_Osmosis_Binary))
 		{
-			echo "settings/configuration.txt already exists\n";
+			echo "Please download osmosis.\nIf it is already installed, check the path in your local settings (settings/local.php) file.\n";
+			if (!$aCMDResult['all'])
+			{
+				fail("osmosis not found in '".CONST_Osmosis_Binary."'");
+			}
 		}
 		else
 		{
-			passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
-			// server layout changed afer license change, fix path to minutely diffs
-			passthru("sed -i 's:minute-replicate:replication/minute:' ".CONST_BasePath.'/settings/configuration.txt');
+			if (file_exists(CONST_BasePath.'/settings/configuration.txt'))
+			{
+				echo "settings/configuration.txt already exists\n";
+			}
+			else
+			{
+				passthru(CONST_Osmosis_Binary.' --read-replication-interval-init '.CONST_BasePath.'/settings');
+				// update osmosis configuration.txt with our settings
+				passthru("sed -i 's!baseUrl=.*!baseUrl=".CONST_Replication_Url."!' ".CONST_BasePath.'/settings/configuration.txt');
+				passthru("sed -i 's:maxInterval = .*:maxInterval = ".CONST_Replication_MaxInterval.":' ".CONST_BasePath.'/settings/configuration.txt');
+			}
+
+			// Find the last node in the DB
+			$iLastOSMID = $oDB->getOne("select max(id) from planet_osm_nodes");
+
+			// Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
+			$sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID."/1";
+			$sLastNodeXML = file_get_contents($sLastNodeURL);
+			preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
+			$iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
+
+			// Search for the correct state file - uses file timestamps so need to sort by date descending
+			$sRepURL = CONST_Replication_Url."/";
+			$sRep = file_get_contents($sRepURL."?C=M;O=D");
+			// download.geofabrik.de:    <a href="000/">000/</a></td><td align="right">26-Feb-2013 11:53  </td>
+			// planet.openstreetmap.org: <a href="273/">273/</a>                    22-Mar-2013 07:41    -
+			preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
+			$aPrevRepMatch = false;
+			foreach($aRepMatches as $aRepMatch)
+			{
+				if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
+				$aPrevRepMatch = $aRepMatch;
+			}
+			if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
+
+			$sRepURL .= $aRepMatch[1];
+			$sRep = file_get_contents($sRepURL."?C=M;O=D");
+			preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
+			$aPrevRepMatch = false;
+			foreach($aRepMatches as $aRepMatch)
+			{
+				if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
+				$aPrevRepMatch = $aRepMatch;
+			}
+			if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
+
+			$sRepURL .= $aRepMatch[1];
+			$sRep = file_get_contents($sRepURL."?C=M;O=D");
+			preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a>.*(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
+			$aPrevRepMatch = false;
+			foreach($aRepMatches as $aRepMatch)
+			{
+				if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
+				$aPrevRepMatch = $aRepMatch;
+			}
+			if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
+
+			$sRepURL .= $aRepMatch[1].'.state.txt';
+			echo "Getting state file: $sRepURL\n";
+			$sStateFile = file_get_contents($sRepURL);
+			if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
+			file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
+			echo "Updating DB status\n";
+			pg_query($oDB->connection, 'TRUNCATE import_status');
+			$sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
+			pg_query($oDB->connection, $sSQL);
 		}
-
-		// Find the last node in the DB
-		$iLastOSMID = $oDB->getOne("select max(osm_id) as osm_id from place where osm_type = 'N'");
-
-		// Lookup the timestamp that node was created (less 3 hours for margin for changsets to be closed)
-		$sLastNodeURL = 'http://www.openstreetmap.org/api/0.6/node/'.$iLastOSMID;
-		$sLastNodeXML = file_get_contents($sLastNodeURL);
-		preg_match('#timestamp="(([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z)"#', $sLastNodeXML, $aLastNodeDate);
-		$iLastNodeTimestamp = strtotime($aLastNodeDate[1]) - (3*60*60);
-
-
-		// Search for the correct state file - uses file timestamps
-		$sRepURL = 'http://planet.openstreetmap.org/replication/minute/';
-		$sRep = file_get_contents($sRepURL);
-		preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
-		$aPrevRepMatch = false;
-		foreach($aRepMatches as $aRepMatch)
-		{
-			if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
-			$aPrevRepMatch = $aRepMatch;
-		}
-		if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
-
-		$sRepURL .= $aRepMatch[1];
-		$sRep = file_get_contents($sRepURL);
-		preg_match_all('#<a href="[0-9]{3}/">([0-9]{3}/)</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
-		$aPrevRepMatch = false;
-		foreach($aRepMatches as $aRepMatch)
-		{
-			if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
-			$aPrevRepMatch = $aRepMatch;
-		}
-		if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
-
-		$sRepURL .= $aRepMatch[1];
-		$sRep = file_get_contents($sRepURL);
-		preg_match_all('#<a href="[0-9]{3}.state.txt">([0-9]{3}).state.txt</a> *(([0-9]{2})-([A-z]{3})-([0-9]{4}) ([0-9]{2}):([0-9]{2}))#', $sRep, $aRepMatches, PREG_SET_ORDER);
-		$aPrevRepMatch = false;
-		foreach($aRepMatches as $aRepMatch)
-		{
-			if (strtotime($aRepMatch[2]) < $iLastNodeTimestamp) break;
-			$aPrevRepMatch = $aRepMatch;
-		}
-		if ($aPrevRepMatch) $aRepMatch = $aPrevRepMatch;
-
-		$sRepURL .= $aRepMatch[1].'.state.txt';
-		echo "Getting state file: $sRepURL\n";
-		$sStateFile = file_get_contents($sRepURL);
-		if (!$sStateFile || strlen($sStateFile) > 1000) fail("unable to obtain state file");
-		file_put_contents(CONST_BasePath.'/settings/state.txt', $sStateFile);
-		echo "Updating DB status\n";
-		pg_query($oDB->connection, 'TRUNCATE import_status');
-		$sSQL = "INSERT INTO import_status VALUES('".$aRepMatch[2]."')";
-		pg_query($oDB->connection, $sSQL);
 	}
 
 	if ($aCMDResult['index'] || $aCMDResult['all'])
@@ -517,7 +589,7 @@
 		$bDidSomething = true;
 		$sOutputFile = '';
 		if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
-		$sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
+		$sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
 		passthruCheckReturn($sBaseCmd.' -R 4');
 		if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
 		passthruCheckReturn($sBaseCmd.' -r 5 -R 25');
@@ -530,13 +602,13 @@
 		echo "Search indices\n";
 		$bDidSomething = true;
 		$oDB =& getDB();
-		$sSQL = 'select partition from country_name order by country_code';
+		$sSQL = 'select distinct partition from country_name';
 		$aPartitions = $oDB->getCol($sSQL);
 		if (PEAR::isError($aPartitions))
 		{
 			fail($aPartitions->getMessage());
 		}
-		$aPartitions[] = 0;
+		if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
 
 		$sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
 		preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
@@ -567,9 +639,20 @@
 		@symlink(CONST_BasePath.'/website/reverse.php', $sTargetDir.'/reverse.php');
 		@symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/search.php');
 		@symlink(CONST_BasePath.'/website/search.php', $sTargetDir.'/index.php');
+		@symlink(CONST_BasePath.'/website/deletable.php', $sTargetDir.'/deletable.php');
+		@symlink(CONST_BasePath.'/website/polygons.php', $sTargetDir.'/polygons.php');
+		@symlink(CONST_BasePath.'/website/status.php', $sTargetDir.'/status.php');
 		@symlink(CONST_BasePath.'/website/images', $sTargetDir.'/images');
 		@symlink(CONST_BasePath.'/website/js', $sTargetDir.'/js');
+		@symlink(CONST_BasePath.'/website/css', $sTargetDir.'/css');
 		echo "Symlinks created\n";
+
+		$sTestFile = @file_get_contents(CONST_Website_BaseURL.'js/tiles.js');
+		if (!$sTestFile)
+		{
+			echo "\nWARNING: Unable to access the website at ".CONST_Website_BaseURL."\n";
+			echo "You may want to update settings/local.php with @define('CONST_Website_BaseURL', 'http://[HOST]/[PATH]/');\n";
+		}
 	}
 
 	if (!$bDidSomething)

utils/update.php

@@ -26,6 +26,7 @@
 
 		array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
 		array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
+		array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
 
 		array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
 		array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
@@ -48,6 +49,7 @@
 
 	if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
 	if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
+
 /*
 	// Lock to prevent multiple copies running
 	if (exec('/bin/ps uww | grep '.basename(__FILE__).' | grep -v /dev/null | grep -v grep -c', $aOutput2, $iResult) > 1)
@@ -67,10 +69,26 @@
 	// Assume osm2pgsql is in the folder above
 	$sBasePath = dirname(dirname(__FILE__));
 
+	date_default_timezone_set('Etc/UTC');
+
 	$oDB =& getDB();
 
 	$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
 
+	// cache memory to be used by osm2pgsql, should not be more than the available memory
+	$iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
+	if ($iCacheMemory + 500 > getTotalMemoryMB())
+	{
+		$iCacheMemory = getCacheMemoryMB();
+		echo "WARNING: resetting cache memory to $iCacheMemory\n";
+	}
+	$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'];
+	if (!is_null(CONST_Osm2pgsql_Flatnode_File))
+	{
+		$sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
+	}
+
+
 	$bFirst = true;
 	$bContinue = $aResult['import-all'];
 	while ($bContinue || $bFirst)
@@ -101,7 +119,7 @@
 			$sNextFile = $aResult['import-diff'];
 			if (!file_exists($sNextFile))
 			{
-				echo "Cannot open $nextFile\n";
+				echo "Cannot open $sNextFile\n";
 				exit;
 			}
 			// Don't update the import status - we don't know what this file contains
@@ -112,7 +130,7 @@
 		if (($aResult['import-hourly'] || $aResult['import-daily'] || isset($aResult['import-diff'])) && file_exists($sNextFile))
 		{
 			// Import the file
-			$sCMD = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$aDSNInfo['database'].' '.$sNextFile;
+			$sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
 			echo $sCMD."\n";
 			exec($sCMD, $sJunk, $iErrorLevel);
 
@@ -147,7 +165,7 @@
 		}
 		else
 		{
-			$sModifyXMLstr = file_get_contents('http://overpass.osm.rambler.ru/cgi/interpreter?data=node('.$aResult['import-node'].');out%20meta;');
+			$sModifyXMLstr = file_get_contents('http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;');
 		}
 	}
 	if (isset($aResult['import-way']) && $aResult['import-way'])
@@ -159,7 +177,7 @@
 		}
 		else
 		{
-			$sCmd = 'http://overpass.osm.rambler.ru/cgi/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
+			$sCmd = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
 		}
 		$sModifyXMLstr = file_get_contents($sCmd);
 	}
@@ -172,7 +190,7 @@
 		}
 		else
 		{
-			$sModifyXMLstr = file_get_contents('http://overpass.osm.rambler.ru/cgi/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;');
+			$sModifyXMLstr = file_get_contents('http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;');
 		}
 	}
 	if ($bModifyXML)
@@ -223,7 +241,7 @@
 		}
 
 		// import generated change file
-		$sCMD = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$aDSNInfo['database'].' '.$sTemporaryFile;
+		$sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
 		echo $sCMD."\n";
 		exec($sCMD, $sJunk, $iErrorLevel);
 		if ($iErrorLevel)
@@ -347,11 +365,18 @@
 
 	if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
 	{
+
+		if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
+			echo "Error: Update interval too low for download.geofabrik.de.  Please check install documentation (http://wiki.openstreetmap.org/wiki/Nominatim/Installation#Updates)\n";
+			exit;
+		}
+
 		$sImportFile = CONST_BasePath.'/data/osmosischange.osc';
 		$sOsmosisCMD = CONST_Osmosis_Binary;
 		$sOsmosisConfigDirectory = CONST_BasePath.'/settings';
 		$sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
-		$sCMDImport = CONST_Osm2pgsql_Binary.' -klas -C 2000 -O gazetteer -d '.$aDSNInfo['database'].' '.$sImportFile;
+		$sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
+		$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
 		$sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'];
 		if (!$aResult['no-npi']) {
 			$sCMDIndex .= '-F ';
@@ -370,11 +395,34 @@
 //			{
 				if (!file_exists($sImportFile))
 				{
-					// Use osmosis to download the file
+					// First check if there are new updates published (except for minutelies - there's always new diffs to process)
+					if ( CONST_Replication_Update_Interval > 60 )
+					{
+
+						unset($aReplicationLag);
+						exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+						while ($iErrorLevel > 0 || $aReplicationLag[0] < 1)
+						{
+							if ($iErrorLevel)
+							{
+								echo "Error: $iErrorLevel. ";
+								echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
+							}
+							else
+							{
+								echo ".";
+							}
+							sleep(CONST_Replication_Recheck_Interval);
+							unset($aReplicationLag);
+							exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
+						}
+						// There are new replication files - use osmosis to download the file
+						echo "\nReplication Delay is ".$aReplicationLag[0]."\n";
+					}
 					$fCMDStartTime = time();
 					echo $sCMDDownload."\n";
 					exec($sCMDDownload, $sJunk, $iErrorLevel);
-					while ($iErrorLevel == 1)
+					while ($iErrorLevel > 0)
 					{
 						echo "Error: $iErrorLevel\n";
 						sleep(60);
@@ -398,7 +446,7 @@
 				if ($iErrorLevel)
 				{
 					echo "Error: $iErrorLevel\n";
-					exit;
+					exit($iErrorLevel);
 				}
 				echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 				$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
@@ -421,7 +469,7 @@
 				if (PEAR::isError($iFileID))
 				{
 					echo $iFileID->getMessage()."\n";
-					exit;
+					exit(-1);
 				} 
 				$sFileDir = CONST_BasePath.'/export/diff/';
 				$sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT);
@@ -447,7 +495,7 @@
 				if ($iErrorLevel)
 				{
 					echo "Error: $iErrorLevel\n";
-					exit;
+					exit($iErrorLevel);
 				}
 
 				if (!$aResult['no-npi'])
@@ -462,7 +510,7 @@
 					if ($iErrorLevel)
 					{
 						echo "Error: $iErrorLevel\n";
-						exit;
+						exit($iErrorLevel);
 					}
 
 					rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2",
@@ -481,8 +529,16 @@
 			echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n";
 			if (!$aResult['import-osmosis-all']) exit;
 
-			echo "Sleeping ".max(0,60-$fDuration)." seconds\n";
-			sleep(max(0,60-$fDuration));
+			if ( CONST_Replication_Update_Interval > 60 )
+			{
+				$iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
+			}
+			else
+			{
+				$iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
+			}
+			echo "Sleeping $iSleep seconds\n";
+			sleep($iSleep);
 		}
 
 	}

website/css/details.css

@@ -0,0 +1,26 @@
+body {
+    margin:0px;
+    padding:16px;
+  background:#ffffff;
+  height: 100%;
+  font: normal 12px/15px arial,sans-serif;
+}
+.line{
+  margin-left:20px;
+}
+.name{
+  font-weight: bold;
+}
+.notused{
+  color:#ddd;
+}
+.noname{
+  color:#800;
+}
+#map {
+  width:500px;
+  height:500px;
+  border: 2px solid #666;
+  float: right;
+}
+

website/css/search.css

@@ -0,0 +1,166 @@
+* {-moz-box-sizing: border-box;}
+body {
+  margin:0px;
+  padding:0px;
+  overflow: hidden;
+  background:#ffffff;
+  height: 100%;
+  font: normal 12px/15px arial,sans-serif;
+}
+#seachheader {
+  position:absolute;
+  z-index:5;
+  top:0px;
+  left:0px;
+  width:100%;
+  height:38px;
+  background:#F0F7FF;
+  border-bottom: 2px solid #75ADFF;
+}
+#q {
+  width:300px;
+}
+#seachheaderfade1, #seachheaderfade2, #seachheaderfade3, #seachheaderfade4{
+  position:absolute;
+  z-index:4;
+  top:0px;
+  left:0px;
+  width:100%;
+  opacity: 0.15;
+  filter: alpha(opacity = 15);
+  background:#000000;
+  border: 1px solid #000000;
+}
+#seachheaderfade1{
+  height:39px;
+}
+#seachheaderfade2{
+  height:40px;
+}
+#seachheaderfade3{
+  height:41px;
+}
+#seachheaderfade4{
+  height:42px;
+}
+#searchresultsfade1, #searchresultsfade2, #searchresultsfade3, #searchresultsfade4 {
+  position:absolute;
+  z-index:2;
+  top:0px;
+  left:200px;
+  height: 100%;
+  opacity: 0.2;
+  filter: alpha(opacity = 20);
+  background:#ffffff;
+  border: 1px solid #ffffff;
+}
+#searchresultsfade1{
+  width:1px;
+}
+#searchresultsfade2{
+  width:2px;
+}
+#searchresultsfade3{
+  width:3px;
+}
+#searchresultsfade4{
+  width:4px;
+}
+
+#searchresults{
+  position:absolute;
+  z-index:3;
+  top:41px;
+  width:200px;
+  height: 100%;
+  background:#ffffff;
+  border: 1px solid #ffffff;
+  overflow: auto;
+}
+#map{
+  position:absolute;
+  z-index:1;
+  top:38px;
+  left:200px;
+  width:100%;
+  height:100%;
+  background:#eee;
+}
+#report{
+  position:absolute;
+  z-index:2;
+  top:38px;
+  left:200px;
+  width:100%;
+  height:100%;
+  background:#eee;
+  font: normal 12px/15px arial,sans-serif;
+  padding:20px;
+}
+#report table {
+  margin-left:20px;
+}
+#report th {
+  vertical-align:top;
+  text-align:left;
+}
+#report td.button {
+  text-align:right;
+}
+.result {
+  margin:5px;
+  margin-bottom:0px;
+  padding:2px;
+  padding-left:4px;
+  padding-right:4px;
+  border-radius: 5px;
+  -moz-border-radius: 5px;
+  -webkit-border-radius: 5px;
+  background:#F0F7FF;
+  border: 2px solid #D7E7FF;
+  font: normal 12px/15px arial,sans-serif;
+  cursor:pointer;
+}
+.result img{
+  float:right;
+}
+.result .latlon{
+  display: none;
+}
+.result .place_id{
+  display: none;
+}
+.result .type{
+  color: #999;
+  text-align:center;
+  font: normal 9px/10px arial,sans-serif;
+  padding-top:4px;
+}
+.result .details, .result .details a{
+  color: #999;
+  text-align:center;
+  font: normal 9px/10px arial,sans-serif;
+  padding-top:4px;
+}
+.noresults{
+  color: #000;
+  text-align:center;
+  font: normal 12px arial,sans-serif;
+  padding-top:4px;
+}
+.more{
+  color: #ccc;
+  text-align:center;
+  padding-top:4px;
+}
+.disclaimer{
+  color: #ccc;
+  text-align:center;
+  font: normal 9px/10px arial,sans-serif;
+  padding-top:4px;
+}
+form{
+  margin:0px;
+  padding:0px;
+}
+

website/deletable.php

@@ -0,0 +1,102 @@
+<?php
+        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
+        require_once(CONST_BasePath.'/lib/log.php');
+
+	$sOutputFormat = 'html';
+	ini_set('memory_limit', '200M');
+
+	$oDB =& getDB();
+
+	$sSQL = "select placex.place_id, calculated_country_code as country_code, name->'name' as name, i.* from placex, import_polygon_delete i where placex.osm_id = i.osm_id and placex.osm_type = i.osm_type and placex.class = i.class and placex.type = i.type";
+	$aPolygons = $oDB->getAll($sSQL);
+	if (PEAR::isError($aPolygons))
+	{
+		failInternalError("Could not get list of deleted OSM elements.", $sSQL, $aPolygons);
+	}
+
+//var_dump($aPolygons);
+?>
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8"/>
+    <meta http-equiv="Content-Type" content="text/html;charset=utf-8" >
+    
+    <title>Nominatim Deleted Data</title>
+    
+    <meta name="description" content="List of OSM data that has been deleted" lang="en-US" />
+
+</head>
+
+<body>
+<style type="text/css">
+table {
+	border-width: 1px;
+	border-spacing: 0px;
+	border-style: solid;
+	border-color: gray;
+	border-collapse: collapse;
+	background-color: white;
+	margin: 10px;
+}
+table th {
+	border-width: 1px;
+	padding: 2px;
+	border-style: inset;
+	border-color: gray;
+	border-left-color: #ddd;
+	border-right-color: #ddd;
+	background-color: #eee;
+	-moz-border-radius: 0px 0px 0px 0px;
+}
+table td {
+	border-width: 1px;
+	padding: 2px;
+	border-style: inset;
+	border-color: gray;
+	border-left-color: #ddd;
+	border-right-color: #ddd;
+	background-color: white;
+	-moz-border-radius: 0px 0px 0px 0px;
+}
+</style>
+
+<p>Objects in this table have been deleted in OSM but are still in the Nominatim database.</p>
+
+<table>
+<?php
+	echo "<tr>";
+//var_dump($aPolygons[0]);
+	foreach($aPolygons[0] as $sCol => $sVal)
+	{
+		echo "<th>".$sCol."</th>";
+	}
+	echo "</tr>";
+	foreach($aPolygons as $aRow)
+	{
+		echo "<tr>";
+		foreach($aRow as $sCol => $sVal)
+		{
+			switch($sCol)
+			{
+            case 'osm_id':
+                $sOSMType = ($aRow['osm_type'] == 'N'?'node':($aRow['osm_type'] == 'W'?'way':($aRow['osm_type'] == 'R'?'relation':'')));
+                echo '<td><a href="http://www.openstreetmap.org/browse/'.$sOSMType.'/'.$sVal.'" target="_new">'.$sVal.'</a></td>';
+                                break;
+			case 'place_id':
+				echo '<td><a href="'.CONST_Website_BaseURL.'details?place_id='.$sVal.'">'.$sVal.'</a></td>';
+				break;
+			default:
+				echo "<td>".($sVal?$sVal:'&nbsp;')."</td>";
+				break;
+			}
+		}
+		echo "</tr>";
+	}
+?>
+</table>
+
+
+
+</body>
+</html>

website/details.php

@@ -1,23 +1,43 @@
 <?php
-        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
-        require_once(CONST_BasePath.'/lib/log.php');
+	@define('CONST_ConnectionBucket_PageType', 'Details');
+
+	require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
+	require_once(CONST_BasePath.'/lib/log.php');
 
 	$sOutputFormat = 'html';
-/*
-        $fLoadAvg = getLoadAverage();
-        if ($fLoadAvg > 3)
-        {
-		echo "Page temporarily blocked due to high server load\n";
-                exit;
-        }
-*/
+	/*
+	   $fLoadAvg = getLoadAverage();
+	   if ($fLoadAvg > 3)
+	   {
+	   echo "Page temporarily blocked due to high server load\n";
+	   exit;
+	   }
+	 */
 	ini_set('memory_limit', '200M');
 
 	$oDB =& getDB();
 
+	$aLangPrefOrder = getPreferredLanguages();
+	$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]";
+
 	if (isset($_GET['osmtype']) && isset($_GET['osmid']) && (int)$_GET['osmid'] && ($_GET['osmtype'] == 'N' || $_GET['osmtype'] == 'W' || $_GET['osmtype'] == 'R'))
 	{
 		$_GET['place_id'] = $oDB->getOne("select place_id from placex where osm_type = '".$_GET['osmtype']."' and osm_id = ".(int)$_GET['osmid']." order by type = 'postcode' asc");
+
+		// Be nice about our error messages for broken geometry
+		if (!$_GET['place_id'])
+		{
+			$aPointDetails = $oDB->getRow("select osm_type, osm_id, errormessage, class, type, get_name_by_language(name,$sLanguagePrefArraySQL) as localname, ST_AsText(prevgeometry) as prevgeom, ST_AsText(newgeometry) as newgeom from import_polygon_error where osm_type = '".$_GET['osmtype']."' and osm_id = ".(int)$_GET['osmid']." order by updated desc limit 1");
+			if (!PEAR::isError($aPointDetails) && $aPointDetails) {
+				if (preg_match('/\[(-?\d+\.\d+) (-?\d+\.\d+)\]/', $aPointDetails['errormessage'], $aMatches))
+				{
+					$aPointDetails['error_x'] = $aMatches[1];
+					$aPointDetails['error_y'] = $aMatches[2];
+				}
+				include(CONST_BasePath.'/lib/template/details-error-'.$sOutputFormat.'.php');
+				exit;
+			}
+		}
 	}
 
 	if (!isset($_GET['place_id']))
@@ -33,9 +53,6 @@
 	$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
 	if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
 
-	$aLangPrefOrder = getPreferredLanguages();
-	$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]";
-
 	$hLog = logStart($oDB, 'details', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
 
 	// Make sure the point we are reporting on is fully indexed
@@ -43,20 +60,20 @@
 	//$oDB->query($sSQL);
 
 	// Get the details for this point
-	$sSQL = "select place_id, osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, country_code, importance, wikipedia,";
-	$sSQL .= " parent_place_id, rank_address, rank_search, get_searchrank_label(rank_search) as rank_search_label, get_name_by_language(name,$sLanguagePrefArraySQL) as localname, ";
-	$sSQL .= " ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea,ST_GeometryType(geometry) as geotype, ST_Y(ST_Centroid(geometry)) as lat,ST_X(ST_Centroid(geometry)) as lon ";
+	$sSQL = "select place_id, osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, calculated_country_code as country_code, importance, wikipedia,";
+	$sSQL .= " to_char(indexed_date, 'YYYY-MM-DD HH24:MI') as indexed_date, parent_place_id, rank_address, rank_search, get_searchrank_label(rank_search) as rank_search_label, get_name_by_language(name,$sLanguagePrefArraySQL) as localname, ";
+	$sSQL .= " ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, ";
+	//$sSQL .= " ST_Area(geometry::geography) as area, ";
+	$sSQL .= " ST_y(centroid) as lat, ST_x(centroid) as lon,";
+	$sSQL .= " case when importance = 0 OR importance IS NULL then 0.75-(rank_search::float/40) else importance end as calculated_importance, ";
+	$sSQL .= " ST_AsText(CASE WHEN ST_NPoints(geometry) > 5000 THEN ST_SimplifyPreserveTopology(geometry, 0.0001) ELSE geometry END) as outlinestring";
 	$sSQL .= " from placex where place_id = $iPlaceID";
 	$aPointDetails = $oDB->getRow($sSQL);
 	if (PEAR::IsError($aPointDetails))
 	{
 		failInternalError("Could not get details of place object.", $sSQL, $aPointDetails);
 	}
-
-        $aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber'];
-	$fLon = $aPointDetails['lon'];
-	$fLat = $aPointDetails['lat'];
-	$iZoom = 14;
+	$aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber'];
 
 	$aClassType = getClassTypesWithImportance();
 	$aPointDetails['icon'] = $aClassType[$aPointDetails['class'].':'.$aPointDetails['type']]['icon'];
@@ -69,43 +86,6 @@
 	$sSQL = "select (each(extratags)).key,(each(extratags)).value from placex where place_id = $iPlaceID order by (each(extratags)).key";
 	$aPointDetails['aExtraTags'] = $oDB->getAssoc($sSQL);
 
-	// Get the bounding box and outline polygon
-	$sSQL = "select ST_AsText(geometry) as outlinestring,";
-	$sSQL .= "ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,";
-	$sSQL .= "ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon";
-	$sSQL .= " from placex where place_id = $iPlaceID";
-	$aPointPolygon = $oDB->getRow($sSQL);
-	IF (PEAR::IsError($aPointPolygon))
-	{
-		failInternalError("Could not get bounding box of place object.", $sSQL, $aPointPolygon);
-	}
-	if (preg_match('#POLYGON\\(\\(([- 0-9.,]+)#',$aPointPolygon['outlinestring'],$aMatch))
-	{
-		preg_match_all('/(-?[0-9.]+) (-?[0-9.]+)/',$aMatch[1],$aPolyPoints,PREG_SET_ORDER);
-	}
-	elseif (preg_match('#MULTIPOLYGON\\(\\(\\(([- 0-9.,]+)#',$aPointPolygon['outlinestring'],$aMatch))
-	{
-		// TODO: this just takes the first ring
-		preg_match_all('/(-?[0-9.]+) (-?[0-9.]+)/',$aMatch[1],$aPolyPoints,PREG_SET_ORDER);
-	}
-	elseif (preg_match('#POINT\\((-?[0-9.]+) (-?[0-9.]+)\\)#',$aPointPolygon['outlinestring'],$aMatch))
-	{
-		$fRadius = 0.01;
-		if ($aPointDetails['rank_search'] > 20) $fRadius = 0.0001;
-		$iSteps = min(max(($fRadius * 40000)^2,16),100);
-		$fStepSize = (2*pi())/$iSteps;
-		$aPolyPoints = array();
-		for($f = 0; $f < 2*pi(); $f += $fStepSize)
-		{
-
-			$aPolyPoints[] = array('',$aMatch[1]+($fRadius*sin($f)),$aMatch[2]+($fRadius*cos($f)));
-		}
-		$aPointPolygon['minlat'] = $aPointPolygon['minlat'] - $fRadius;
-		$aPointPolygon['maxlat'] = $aPointPolygon['maxlat'] + $fRadius;
-		$aPointPolygon['minlon'] = $aPointPolygon['minlon'] - $fRadius;
-		$aPointPolygon['maxlon'] = $aPointPolygon['maxlon'] + $fRadius;
-	}
-
 	// Address
 	$aAddressLines = getAddressDetails($oDB, $sLanguagePrefArraySQL, $iPlaceID, $aPointDetails['country_code'], true);
 
@@ -118,11 +98,12 @@
 	$aLinkedLines = $oDB->getAll($sSQL);
 
 	// All places this is an imediate parent of
-	$sSQL = "select placex.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, st_distance(geometry, placegeometry) as distance, ";
+	$sSQL = "select obj.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, st_distance(geometry, placegeometry) as distance, ";
 	$sSQL .= " get_name_by_language(name,$sLanguagePrefArraySQL) as localname, length(name::text) as namelength ";
-	$sSQL .= " from placex, (select geometry as placegeometry from placex where place_id = $iPlaceID) as x";
-	$sSQL .= " where parent_place_id = $iPlaceID";
-	$sSQL .= " order by rank_address asc,rank_search asc,get_name_by_language(name,$sLanguagePrefArraySQL),housenumber";
+	$sSQL .= " from (select placex.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, rank_search, geometry, name from placex ";
+	$sSQL .= " where parent_place_id = $iPlaceID order by rank_address asc,rank_search asc limit 500) as obj,";
+	$sSQL .= " (select geometry as placegeometry from placex where place_id = $iPlaceID) as x";
+	$sSQL .= " order by rank_address asc,rank_search asc,localname,housenumber";
 	$aParentOfLines = $oDB->getAll($sSQL);
 
 	$aPlaceSearchNameKeywords = false;

website/hierarchy.php

@@ -0,0 +1,214 @@
+<?php
+	@define('CONST_ConnectionBucket_PageType', 'Details');
+
+	require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
+	require_once(CONST_BasePath.'/lib/log.php');
+	require_once(CONST_BasePath.'/lib/PlaceLookup.php');
+
+	$sOutputFormat = 'html';
+	if (isset($_GET['format']) && ($_GET['format'] == 'html' || $_GET['format'] == 'xml' || $_GET['format'] == 'json' ||  $_GET['format'] == 'jsonv2'))
+	{
+		$sOutputFormat = $_GET['format'];
+	}
+
+	ini_set('memory_limit', '200M');
+
+	$oDB =& getDB();
+
+	$aLangPrefOrder = getPreferredLanguages();
+	$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]";
+
+	if (isset($_GET['osmtype']) && isset($_GET['osmid']) && (int)$_GET['osmid'] && ($_GET['osmtype'] == 'N' || $_GET['osmtype'] == 'W' || $_GET['osmtype'] == 'R'))
+	{
+		$_GET['place_id'] = $oDB->getOne("select place_id from placex where osm_type = '".$_GET['osmtype']."' and osm_id = ".(int)$_GET['osmid']." order by type = 'postcode' asc");
+
+		// Be nice about our error messages for broken geometry
+		if (!$_GET['place_id'])
+		{
+			$aPointDetails = $oDB->getRow("select osm_type, osm_id, errormessage, class, type, get_name_by_language(name,$sLanguagePrefArraySQL) as localname, ST_AsText(prevgeometry) as prevgeom, ST_AsText(newgeometry) as newgeom from import_polygon_error where osm_type = '".$_GET['osmtype']."' and osm_id = ".(int)$_GET['osmid']." order by updated desc limit 1");
+			if (!PEAR::isError($aPointDetails) && $aPointDetails) {
+				if (preg_match('/\[(-?\d+\.\d+) (-?\d+\.\d+)\]/', $aPointDetails['errormessage'], $aMatches))
+				{
+					$aPointDetails['error_x'] = $aMatches[1];
+					$aPointDetails['error_y'] = $aMatches[2];
+				}
+				include(CONST_BasePath.'/lib/template/details-error-'.$sOutputFormat.'.php');
+				exit;
+			}
+		}
+	}
+
+	if (!isset($_GET['place_id']))
+	{
+		echo "Please select a place id";
+		exit;
+	}
+
+	$iPlaceID = (int)$_GET['place_id'];
+
+	$sAuxHouseNumber = false;
+	$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
+	if ($iParentPlaceID)
+	{
+		$iPlaceID = $iParentPlaceID;
+	}
+	else
+	{
+		$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
+		if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+	}
+
+	$oPlaceLookup = new PlaceLookup($oDB);
+	$oPlaceLookup->setLanguagePreference($aLangPrefOrder);
+	$oPlaceLookup->setIncludeAddressDetails(true);
+	$oPlaceLookup->setPlaceId($iPlaceID);
+
+	$aPlaceAddress = array_reverse($oPlaceLookup->getAddressDetails());
+	$aBreadcrums = array();
+	foreach($aPlaceAddress as $i => $aPlace)
+	{
+		if (!$aPlace['place_id']) continue;
+		$aBreadcrums[] = array('placeId'=>$aPlace['place_id'], 'osmType'=>$aPlace['osm_type'], 'osmId'=>$aPlace['osm_id'], 'localName'=>$aPlace['localname']);
+		$sPlaceUrl = 'hierarchy.php?place_id='.$aPlace['place_id'];
+		$sOSMType = ($aPlace['osm_type'] == 'N'?'node':($aPlace['osm_type'] == 'W'?'way':($aPlace['osm_type'] == 'R'?'relation':'')));
+                $sOSMUrl = 'http://www.openstreetmap.org/browse/'.$sOSMType.'/'.$aPlace['osm_id'];
+		if ($sOutputFormat == 'html') if ($i) echo " > ";
+		if ($sOutputFormat == 'html') echo '<a href="'.$sPlaceUrl.'">'.$aPlace['localname'].'</a> (<a href="'.$sOSMUrl.'">osm</a>)';
+	}
+
+	$aDetails = array();
+	$aDetails['breadcrumbs'] = $aBreadcrums;
+
+	if ($sOutputFormat == 'json')
+	{
+		header("content-type: application/json; charset=UTF-8");
+		javascript_renderData($aDetails);
+		exit;
+	}
+
+	$aRelatedPlaceIDs = $oDB->getCol($sSQL = "select place_id from placex where linked_place_id = $iPlaceID or place_id = $iPlaceID");
+
+	$sSQL = "select obj.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea,  st_area(geometry) as area, ";
+	$sSQL .= " get_name_by_language(name,$sLanguagePrefArraySQL) as localname, length(name::text) as namelength ";
+	$sSQL .= " from (select placex.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, rank_search, geometry, name from placex ";
+	$sSQL .= " where parent_place_id in (".join(',',$aRelatedPlaceIDs).") and name is not null order by rank_address asc,rank_search asc limit 500) as obj";
+	$sSQL .= " order by rank_address asc,rank_search asc,localname,class, type,housenumber";
+	$aParentOfLines = $oDB->getAll($sSQL);
+
+        if (sizeof($aParentOfLines))
+        {
+                echo '<h2>Parent Of:</h2>';
+		$aClassType = getClassTypesWithImportance();
+                $aGroupedAddressLines = array();
+                foreach($aParentOfLines as $aAddressLine)
+                {
+			if (isset($aClassType[$aAddressLine['class'].':'.$aAddressLine['type'].':'.$aAddressLine['admin_level']]['label'])
+                                              && $aClassType[$aAddressLine['class'].':'.$aAddressLine['type'].':'.$aAddressLine['admin_level']]['label'])
+                                {
+                                        $aAddressLine['label'] = $aClassType[$aAddressLine['class'].':'.$aAddressLine['type'].':'.$aAddressLine['admin_level']]['label'];
+                                }
+			elseif (isset($aClassType[$aAddressLine['class'].':'.$aAddressLine['type']]['label'])
+                                              && $aClassType[$aAddressLine['class'].':'.$aAddressLine['type']]['label'])
+                                {
+                                        $aAddressLine['label'] = $aClassType[$aAddressLine['class'].':'.$aAddressLine['type']]['label'];
+                                }
+			else $aAddressLine['label'] = ucwords($aAddressLine['type']);
+
+                        if (!isset($aGroupedAddressLines[$aAddressLine['label']])) $aGroupedAddressLines[$aAddressLine['label']] = array();
+                        $aGroupedAddressLines[$aAddressLine['label']][] = $aAddressLine;
+                }
+                foreach($aGroupedAddressLines as $sGroupHeading => $aParentOfLines)
+                {
+                        echo "<h3>$sGroupHeading</h3>";
+                foreach($aParentOfLines as $aAddressLine)
+                {
+                        $aAddressLine['localname'] = $aAddressLine['localname']?$aAddressLine['localname']:$aAddressLine['housenumber'];
+                        $sOSMType = ($aAddressLine['osm_type'] == 'N'?'node':($aAddressLine['osm_type'] == 'W'?'way':($aAddressLine['osm_type'] == 'R'?'relation':'')));
+
+                        echo '<div class="line">';
+                        echo '<span class="name">'.(trim($aAddressLine['localname'])?$aAddressLine['localname']:'<span class="noname">No Name</span>').'</span>';
+                        echo ' (';
+                        echo '<span class="area">'.($aAddressLine['isarea']=='t'?'Polygon':'Point').'</span>';
+                        if ($sOSMType) echo ', <span class="osm"><span class="label"></span>'.$sOSMType.' <a href="http://www.openstreetmap.org/browse/'.$sOSMType.'/'.$aAddressLine['osm_id'].'">'.$aAddressLine['osm_id'].'</a></span>';
+                        echo ', <a href="hierarchy.php?place_id='.$aAddressLine['place_id'].'">GOTO</a>';
+                        echo ', '.$aAddressLine['area'];
+                        echo ')';
+                        echo '</div>';
+                }
+                }
+                if (sizeof($aParentOfLines) >= 500) {
+                        echo '<p>There are more child objects which are not shown.</p>';
+                }
+                echo '</div>';
+        }
+exit;
+
+
+	$hLog = logStart($oDB, 'details', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
+
+	// Make sure the point we are reporting on is fully indexed
+	//$sSQL = "UPDATE placex set indexed = true where indexed = false and place_id = $iPlaceID";
+	//$oDB->query($sSQL);
+
+	// Get the details for this point
+	$sSQL = "select place_id, osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, calculated_country_code as country_code, importance, wikipedia,";
+	$sSQL .= " to_char(indexed_date, 'YYYY-MM-DD HH24:MI') as indexed_date, parent_place_id, rank_address, rank_search, get_searchrank_label(rank_search) as rank_search_label, get_name_by_language(name,$sLanguagePrefArraySQL) as localname, ";
+	$sSQL .= " ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, ";
+	//$sSQL .= " ST_Area(geometry::geography) as area, ";
+	$sSQL .= " ST_y(centroid) as lat, ST_x(centroid) as lon,";
+	$sSQL .= " case when importance = 0 OR importance IS NULL then 0.75-(rank_search::float/40) else importance end as calculated_importance, ";
+	$sSQL .= " ST_AsText(CASE WHEN ST_NPoints(geometry) > 5000 THEN ST_SimplifyPreserveTopology(geometry, 0.0001) ELSE geometry END) as outlinestring";
+	$sSQL .= " from placex where place_id = $iPlaceID";
+	$aPointDetails = $oDB->getRow($sSQL);
+	if (PEAR::IsError($aPointDetails))
+	{
+		failInternalError("Could not get details of place object.", $sSQL, $aPointDetails);
+	}
+	$aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber'];
+
+	$aClassType = getClassTypesWithImportance();
+	$aPointDetails['icon'] = $aClassType[$aPointDetails['class'].':'.$aPointDetails['type']]['icon'];
+
+	// Get all alternative names (languages, etc)
+	$sSQL = "select (each(name)).key,(each(name)).value from placex where place_id = $iPlaceID order by (each(name)).key";
+	$aPointDetails['aNames'] = $oDB->getAssoc($sSQL);
+
+	// Extra tags
+	$sSQL = "select (each(extratags)).key,(each(extratags)).value from placex where place_id = $iPlaceID order by (each(extratags)).key";
+	$aPointDetails['aExtraTags'] = $oDB->getAssoc($sSQL);
+
+	// Address
+	$aAddressLines = getAddressDetails($oDB, $sLanguagePrefArraySQL, $iPlaceID, $aPointDetails['country_code'], true);
+
+	// Linked places
+	$sSQL = "select placex.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, st_distance(geometry, placegeometry) as distance, ";
+	$sSQL .= " get_name_by_language(name,$sLanguagePrefArraySQL) as localname, length(name::text) as namelength ";
+	$sSQL .= " from placex, (select geometry as placegeometry from placex where place_id = $iPlaceID) as x";
+	$sSQL .= " where linked_place_id = $iPlaceID";
+	$sSQL .= " order by rank_address asc,rank_search asc,get_name_by_language(name,$sLanguagePrefArraySQL),housenumber";
+	$aLinkedLines = $oDB->getAll($sSQL);
+
+	// All places this is an imediate parent of
+	$sSQL = "select obj.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, st_distance(geometry, placegeometry) as distance, ";
+	$sSQL .= " get_name_by_language(name,$sLanguagePrefArraySQL) as localname, length(name::text) as namelength ";
+	$sSQL .= " from (select placex.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, rank_search, geometry, name from placex ";
+	$sSQL .= " where parent_place_id = $iPlaceID order by rank_address asc,rank_search asc limit 500) as obj,";
+	$sSQL .= " (select geometry as placegeometry from placex where place_id = $iPlaceID) as x";
+	$sSQL .= " order by rank_address asc,rank_search asc,localname,housenumber";
+	$aParentOfLines = $oDB->getAll($sSQL);
+
+	$aPlaceSearchNameKeywords = false;
+	$aPlaceSearchAddressKeywords = false;
+	if (isset($_GET['keywords']) && $_GET['keywords'])
+	{
+		$sSQL = "select * from search_name where place_id = $iPlaceID";
+		$aPlaceSearchName = $oDB->getRow($sSQL);
+		$sSQL = "select * from word where word_id in (".substr($aPlaceSearchName['name_vector'],1,-1).")";
+		$aPlaceSearchNameKeywords = $oDB->getAll($sSQL);
+		$sSQL = "select * from word where word_id in (".substr($aPlaceSearchName['nameaddress_vector'],1,-1).")";
+		$aPlaceSearchAddressKeywords = $oDB->getAll($sSQL);
+	}
+
+	logEnd($oDB, $hLog, 1);
+
+	include(CONST_BasePath.'/lib/template/details-'.$sOutputFormat.'.php');

website/js/tiles.js

@@ -48,7 +48,8 @@ OpenLayers.Layer.OSM.Mapnik = OpenLayers.Class(OpenLayers.Layer.OSM, {
             "http://b.tile.openstreetmap.org/${z}/${x}/${y}.png",
             "http://c.tile.openstreetmap.org/${z}/${x}/${y}.png"
         ];
-        options = OpenLayers.Util.extend({ numZoomLevels: 19, buffer: 0 }, options);
+        options = OpenLayers.Util.extend({ numZoomLevels: 19, buffer: 0,
+           attribution : '© <a target="_parent" href="http://www.openstreetmap.org">OpenStreetMap</a> and contributors, under an <a target="_parent" href="http://www.openstreetmap.org/copyright">open license</a>' }, options);
         var newArguments = [name, url, options];
         OpenLayers.Layer.OSM.prototype.initialize.apply(this, newArguments);
     },
@@ -85,34 +86,6 @@ OpenLayers.Layer.OSM.MapQuestOpen = OpenLayers.Class(OpenLayers.Layer.OSM, {
     CLASS_NAME: "OpenLayers.Layer.OSM.MapQuestOpen"
 });
 
-/**
- * Class: OpenLayers.Layer.OSM.Osmarender
- *
- * Inherits from:
- *  - <OpenLayers.Layer.OSM>
- */
-OpenLayers.Layer.OSM.Osmarender = OpenLayers.Class(OpenLayers.Layer.OSM, {
-    /**
-     * Constructor: OpenLayers.Layer.OSM.Osmarender
-     *
-     * Parameters:
-     * name - {String}
-     * options - {Object} Hashtable of extra options to tag onto the layer
-     */
-    initialize: function(name, options) {
-        var url = [
-            "http://a.tah.openstreetmap.org/Tiles/tile/${z}/${x}/${y}.png",
-            "http://b.tah.openstreetmap.org/Tiles/tile/${z}/${x}/${y}.png",
-            "http://c.tah.openstreetmap.org/Tiles/tile/${z}/${x}/${y}.png"
-        ];
-        options = OpenLayers.Util.extend({ numZoomLevels: 18, buffer: 0 }, options);
-        var newArguments = [name, url, options];
-        OpenLayers.Layer.OSM.prototype.initialize.apply(this, newArguments);
-    },
-
-    CLASS_NAME: "OpenLayers.Layer.OSM.Osmarender"
-});
-
 /**
  * Class: OpenLayers.Layer.OSM.CycleMap
  *

website/polygons.php

@@ -1,6 +1,6 @@
 <?php
-        require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
-        require_once(CONST_BasePath.'/lib/log.php');
+	require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
+	require_once(CONST_BasePath.'/lib/log.php');
 
 	$sOutputFormat = 'html';
 	ini_set('memory_limit', '200M');

website/reverse.php

@@ -1,163 +1,65 @@
 <?php
+	@define('CONST_ConnectionBucket_PageType', 'Reverse');
+
 	require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
 	require_once(CONST_BasePath.'/lib/log.php');
+	require_once(CONST_BasePath.'/lib/PlaceLookup.php');
+	require_once(CONST_BasePath.'/lib/ReverseGeocode.php');
 
-        if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
-        {
-                $fLoadAvg = getLoadAverage();
-                if ($fLoadAvg > 2) sleep(60);
-                if ($fLoadAvg > 4) sleep(120);
-                if ($fLoadAvg > 6)
-                {
-                        echo "Bulk User: Temporary block due to high server load\n";
-                        exit;
-                }
-        }
+	if (strpos(CONST_BulkUserIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false)
+	{
+		$fLoadAvg = getLoadAverage();
+		if ($fLoadAvg > 2) sleep(60);
+		if ($fLoadAvg > 4) sleep(120);
+		if ($fLoadAvg > 6)
+		{
+			echo "Bulk User: Temporary block due to high server load\n";
+			exit;
+		}
+	}
 
-        $oDB =& getDB();
-        ini_set('memory_limit', '200M');
+	$oDB =& getDB();
+	ini_set('memory_limit', '200M');
 
-        // Format for output
-        $sOutputFormat = 'xml';
-        if (isset($_GET['format']) && ($_GET['format'] == 'xml' || $_GET['format'] == 'json' || $_GET['format'] == 'jsonv2'))
-        {
-                $sOutputFormat = $_GET['format'];
-        }
+	// Format for output
+	$sOutputFormat = 'xml';
+	if (isset($_GET['format']) && ($_GET['format'] == 'xml' || $_GET['format'] == 'json' || $_GET['format'] == 'jsonv2'))
+	{
+		$sOutputFormat = $_GET['format'];
+	}
 
 	// Show address breakdown
 	$bShowAddressDetails = true;
 	if (isset($_GET['addressdetails'])) $bShowAddressDetails = (bool)$_GET['addressdetails'];
 
-        // Preferred language
-        $aLangPrefOrder = getPreferredLanguages();
-        $sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted",$aLangPrefOrder))."]";
+	// Preferred language
+	$aLangPrefOrder = getPreferredLanguages();
 
 	$hLog = logStart($oDB, 'reverse', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
 
-        if (isset($_GET['osm_type']) && isset($_GET['osm_id']) && (int)$_GET['osm_id'] && ($_GET['osm_type'] == 'N' || $_GET['osm_type'] == 'W' || $_GET['osm_type'] == 'R'))
-        {
-                $iPlaceID = $oDB->getOne("select place_id from placex where osm_type = '".$_GET['osm_type']."' and osm_id = ".(int)$_GET['osm_id']." order by type = 'postcode' asc");
-		if (!$iPlaceID) $sError = 'OSM ID Not Found';
-        }
+	if (isset($_GET['osm_type']) && isset($_GET['osm_id']) && (int)$_GET['osm_id'] && ($_GET['osm_type'] == 'N' || $_GET['osm_type'] == 'W' || $_GET['osm_type'] == 'R'))
+	{
+		$oPlaceLookup = new PlaceLookup($oDB);
+		$oPlaceLookup->setLanguagePreference($aLangPrefOrder);
+		$oPlaceLookup->setIncludeAddressDetails($bShowAddressDetails);
+		$oPlaceLookup->setOSMID($_GET['osm_type'], $_GET['osm_id']);
+
+		$aPlace = $oPlaceLookup->lookup();
+
+		//if (!$iPlaceID) $sError = 'OSM ID Not Found';
+	}
 	else
 	{
-		// Location to look up
-		$fLat = (float)$_GET['lat'];
-		$fLon = (float)$_GET['lon'];
-		$sPointSQL = "ST_SetSRID(ST_Point($fLon,$fLat),4326)";
+		$oReverseGeocode = new ReverseGeocode($oDB);
+		$oReverseGeocode->setLanguagePreference($aLangPrefOrder);
+		$oReverseGeocode->setIncludeAddressDetails($bShowAddressDetails);
 
-		// Zoom to rank, this could probably be calculated but a lookup gives fine control
-		$aZoomRank = array(
-			0 => 2, // Continent / Sea
-			1 => 2,
-			2 => 2,
-			3 => 4, // Country
-			4 => 4,
-			5 => 8, // State
-			6 => 10, // Region
-			7 => 10,
-			8 => 12, // County
-			9 => 12,
-			10 => 17, // City
-			11 => 17,
-			12 => 18, // Town / Village
-			13 => 18,
-			14 => 22, // Suburb
-			15 => 22,
-			16 => 26, // Street, TODO: major street?
-			17 => 26,
-			18 => 30, // or >, Building
-			19 => 30, // or >, Building
-			);
-		$iMaxRank = (isset($_GET['zoom']) && isset($aZoomRank[$_GET['zoom']]))?$aZoomRank[$_GET['zoom']]:28;
+		$oReverseGeocode->setLatLon($_GET['lat'], $_GET['lon']);
+		$oReverseGeocode->setZoom(@$_GET['zoom']);
 
-		// Find the nearest point
-		$fSearchDiam = 0.0001;
-		$iPlaceID = null;
-		$aArea = false;
-		$fMaxAreaDistance = 1;
-		while(!$iPlaceID && $fSearchDiam < $fMaxAreaDistance)
-		{
-			$fSearchDiam = $fSearchDiam * 2;
-
-			// If we have to expand the search area by a large amount then we need a larger feature
-			// then there is a limit to how small the feature should be
-			if ($fSearchDiam > 2 && $iMaxRank > 4) $iMaxRank = 4;
-			if ($fSearchDiam > 1 && $iMaxRank > 9) $iMaxRank = 8;
-			if ($fSearchDiam > 0.8 && $iMaxRank > 10) $iMaxRank = 10;
-			if ($fSearchDiam > 0.6 && $iMaxRank > 12) $iMaxRank = 12;
-			if ($fSearchDiam > 0.2 && $iMaxRank > 17) $iMaxRank = 17;
-			if ($fSearchDiam > 0.1 && $iMaxRank > 18) $iMaxRank = 18;
-			if ($fSearchDiam > 0.008 && $iMaxRank > 22) $iMaxRank = 22;
-			if ($fSearchDiam > 0.001 && $iMaxRank > 26) $iMaxRank = 26;
-
-			$sSQL = 'select place_id,parent_place_id,rank_search from placex';
-			$sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')';
-			$sSQL .= ' and rank_search != 28 and rank_search >= '.$iMaxRank;
-			$sSQL .= ' and (name is not null or housenumber is not null)';
-			$sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\')';
-			$sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') ';
-			$sSQL .= ' OR ST_DWithin('.$sPointSQL.', ST_Centroid(geometry), '.$fSearchDiam.'))';
-			$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1';
-//var_dump($sSQL);
-			$aPlace = $oDB->getRow($sSQL);
-			if (PEAR::IsError($aPlace))
-			{
-				failInternalError("Could not determine closest place.", $sSQL, $iPlaceID); 
-			}
-			$iPlaceID = $aPlace['place_id'];
-			$iParentPlaceID = $aPlace['parent_place_id'];
-		}
-
-		// The point we found might be too small - use the address to find what it is a child of
-		if ($iPlaceID && $iMaxRank < 28)
-		{
-			if ($aPlace['rank_search'] > 28 && $iParentPlaceID) {
-				$iPlaceID = $iParentPlaceID;
-			}
-			$sSQL = "select address_place_id from place_addressline where place_id = $iPlaceID order by abs(cached_rank_address - $iMaxRank) asc,cached_rank_address desc,isaddress desc,distance desc limit 1";
-			$iPlaceID = $oDB->getOne($sSQL);
-			if (PEAR::IsError($iPlaceID))
-			{
-				failInternalError("Could not get parent for place.", $sSQL, $iPlaceID); 
-			}
-			if (!$iPlaceID)
-			{
-				$iPlaceID = $aPlace['place_id'];
-			}
-		}
+		$aPlace = $oReverseGeocode->lookup();
 	}
 
-	if ($iPlaceID)
-	{
-		$sSQL = "select placex.*,";
-        	$sSQL .= " get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
-	        $sSQL .= " get_name_by_language(name, $sLanguagePrefArraySQL) as placename,";
-        	$sSQL .= " get_name_by_language(name, ARRAY['ref']) as ref,";
-        	$sSQL .= " st_y(st_centroid(geometry)) as lat, st_x(st_centroid(geometry)) as lon";
-	        $sSQL .= " from placex where place_id = $iPlaceID ";
-//var_dump($sSQL);
-		$aPlace = $oDB->getRow($sSQL);
+	if (CONST_Debug) exit;
 
-		if ($bShowAddressDetails)
-		{
-			$aAddress = getAddressDetails($oDB, $sLanguagePrefArraySQL, $iPlaceID, $aPlace['country_code']);
-		}
-		$aClassType = getClassTypes();
-		$sAddressType = '';
-		$sClassType = $aPlace['class'].':'.$aPlace['type'].':'.$aPlace['admin_level'];
-		if (isset($aClassType[$sClassType]) && isset($aClassType[$sClassType]['simplelabel']))
-		{
-			$sAddressType = $aClassType[$aClassType]['simplelabel'];
-		}
-		else
-		{
-			$sClassType = $aPlace['class'].':'.$aPlace['type'];
-			if (isset($aClassType[$sClassType]) && isset($aClassType[$sClassType]['simplelabel']))
-				$sAddressType = $aClassType[$sClassType]['simplelabel'];
-			else $sAddressType = $aPlace['class'];
-		}
-		$aPlace['addresstype'] = $sAddressType;
-
-	}
 	include(CONST_BasePath.'/lib/template/address-'.$sOutputFormat.'.php');

website/status.php

@@ -0,0 +1,26 @@
+<?php
+	@define('CONST_ConnectionBucket_PageType', 'Status');
+
+	require_once(dirname(dirname(__FILE__)).'/lib/init-website.php');
+
+	$oDB =& getDB();
+	if (!$oDB || PEAR::isError($oDB))
+	{
+		echo "ERROR: No database";
+		exit;
+	}
+
+	$iWordID = $oDB->getOne("select word_id,word_token, word, class, type, country_code, operator, search_name_count from word where word_token in (' a')");
+	if (PEAR::isError($iWordID))
+	{
+		echo "ERROR: Query failed";
+		exit;
+	}
+	if (!$iWordID)
+	{
+		echo "ERROR: No value";
+		exit;
+	}
+	echo "OK";
+	exit;
+

wikidata/create.sql

@@ -0,0 +1,75 @@
+DROP TABLE entity;
+DROP TABLE entity_label;
+DROP TABLE entity_description;
+DROP TABLE entity_alias;
+DROP TABLE entity_link;
+DROP TABLE entity_property;
+
+CREATE TABLE entity (
+    entity_id           bigint,
+    title               text,
+    pid                 bigint,
+    qid                 bigint,
+    datatype            text,
+    CONSTRAINT pk_entity PRIMARY KEY(entity_id)
+);
+
+CREATE TABLE entity_label  (
+    entity_id           bigint,
+    language            text,
+    label               text,
+    CONSTRAINT pk_entity_label PRIMARY KEY(entity_id,language)
+);
+
+CREATE TABLE entity_description  (
+    entity_id           bigint,
+    language            text,
+    description         text,
+    CONSTRAINT pk_entity_description PRIMARY KEY(entity_id,language)
+);
+
+CREATE TABLE entity_alias  (
+    entity_id           bigint,
+    language            text,
+    alias               text,
+    CONSTRAINT pk_entity_alias PRIMARY KEY(entity_id,language,alias)
+);
+
+CREATE TABLE entity_link  (
+    entity_id           bigint,
+    target              text,
+    value               text,
+    CONSTRAINT pk_entity_link PRIMARY KEY(entity_id,target)
+);
+
+CREATE TABLE entity_link_hit  (
+    entity_id           bigint,
+    target              text,
+    value               text,
+    hits		bigint,
+    CONSTRAINT pk_entity_link_hit PRIMARY KEY(entity_id,target)
+);
+
+CREATE TABLE link_hit  (
+    target              text,
+    value               text,
+    hits		bigint,
+    CONSTRAINT pk_link_hit PRIMARY KEY(target,value)
+);
+
+CREATE TABLE entity_property  (
+    entity_id           bigint,
+    order_id            bigint,
+    pid                 bigint,
+    string              text,
+    toqid               bigint,
+    location            geometry,
+    datetime		timestamp with time zone,
+    CONSTRAINT pk_entity_property PRIMARY KEY(entity_id, order_id)
+);
+
+CREATE TABLE import_link_hit  (
+    target              text,
+    value               text,
+    hits		bigint
+);

wikidata/import.sh

@@ -0,0 +1,28 @@
+PSQL=/usr/lib/postgresql/9.2/bin/psql -d wikidata
+
+cat create.sql | $PSQL
+
+cat entity.csv | $PSQL -c "COPY entity from STDIN WITH CSV"
+cat entity_label.csv | $PSQL -c "COPY entity_label from STDIN WITH CSV"
+cat entity_description.csv | $PSQL -c "COPY entity_description from STDIN WITH CSV"
+cat entity_alias.csv | $PSQL -c "COPY entity_alias from STDIN WITH CSV"
+cat entity_link.csv | $PSQL -c "COPY entity_link from STDIN WITH CSV"
+cat entity_property.csv | $PSQL -c "COPY entity_property from STDIN WITH CSV"
+
+$PSQL -c "create index idx_entity_link_target on entity_link using btree (target,value)"
+$PSQL -c "create index idx_entity_qid on entity using btree (qid)"
+$PSQL -c "create table property_label_en as select pid,null::text as label from entity where pid is not null"
+$PSQL -c "update property_label_en set label = x.label from (select pid,label,language from entity join entity_label using (entity_id) where pid is not null and language = 'en') as x where x.pid = property_label_en.pid"
+$PSQL -c "create unique index idx_property_label_en on property_label_en using btree (pid)"
+$PSQL -c "alter table entity add column label_en text"
+$PSQL -c "update entity set label_en = label from entity_label where entity.entity_id = entity_label.entity_id and language = 'en'"
+$PSQL -c "alter table entity add column description_en text"
+$PSQL -c "update entity set description_en = description from entity_description where entity.entity_id = entity_description.entity_id and language = 'en'"
+
+cat totals.txt | $PSQL -c "COPY import_link_hit from STDIN WITH CSV DELIMITER ' '"
+$PSQL -c "truncate link_hit"
+$PSQL -c "insert into link_hit select target||'wiki', replace(catch_decode_url_part(value), '_', ' '), sum(hits) from import_link_hit where replace(catch_decode_url_part(value), '_', ' ') is not null group by target||'wiki', replace(dcatch_decode_url_part(value), '_', ' ')"
+$PSQL -c "truncate entity_link_hit"
+$PSQL -c "insert into entity_link_hit select entity_id, target, value, coalesce(hits,0) from entity_link left outer join link_hit using (target, value)"
+$PSQL -c "create table entity_hit as select entity_id,sum(hits) as hits from entity_link_hit group by entity_id"
+$PSQL -c "create unique index idx_entity_hit on entity_hit using btree (entity_id)"

wikidata/parse.php

@@ -0,0 +1,188 @@
+#!/usr/bin/php -Cq
+<?php
+
+$hFile = @fopen("wikidatawiki-20130623-pages-articles.xml", "r");
+
+$hFileEntity = fopen("entity.csv", "w");
+$hFileEntityLabel = fopen("entity_label.csv", "w");
+$hFileEntityDescription = fopen("entity_description.csv", "w");
+$hFileEntityAlias = fopen("entity_alias.csv", "w");
+$hFileEntityLink = fopen("entity_link.csv", "w");
+$hFileEntityProperty = fopen("entity_property.csv", "w");
+
+$iCount = 0;
+
+$sTitle = '';
+$iNS = false;
+$iID = false;
+
+if ($hFile) {
+	while (($sLine = fgets($hFile, 4000000)) !== false) {
+		if (substr($sLine, 0, 11) == '    <title>') {
+			$sTitle = substr($sLine, 11, -9);
+		}
+		else if (substr($sLine, 0, 8) == '    <ns>') {
+			$iNS = (int)substr($sLine, 8, -6);
+		}
+		else if (substr($sLine, 0, 8) == '    <id>') {
+			$iID = (int)substr($sLine, 8, -6);
+		}
+		else if (substr($sLine, 0, 33) == '      <text xml:space="preserve">') {
+			if ($iNS == -2) continue;
+			if ($iNS == -1) continue;
+			if ($iNS == 1) continue;
+			if ($iNS == 2) continue;
+			if ($iNS == 3) continue;
+			if ($iNS == 4) continue;
+			if ($iNS == 5) continue;
+			if ($iNS == 6) continue;
+			if ($iNS == 7) continue;
+			if ($iNS == 8) continue;
+			if ($iNS == 9) continue;
+			if ($iNS == 10) continue;
+			if ($iNS == 11) continue;
+			if ($iNS == 12) continue;
+			if ($iNS == 13) continue;
+			if ($iNS == 14) continue;
+			if ($iNS == 15) continue;
+			if ($iNS == 121) continue;
+			if ($iNS == 123) continue;
+			if ($iNS == 829) continue;
+			if ($iNS == 1198) continue;
+			if ($iNS == 1199) continue;
+			$sText = html_entity_decode(substr($sLine, 33, -8), ENT_COMPAT, 'UTF-8');
+			$aArticle = json_decode($sText, true);
+
+			if (array_diff(array_keys($aArticle), array("label", "description", "aliases", "links", "entity", "claims", "datatype")) != array()) {
+				// DEBUG
+				var_dump($sTitle);
+				var_dump(array_keys($aArticle));
+				var_dump($aArticle);
+				exit;
+			}
+
+			$iPID = $iQID = null;
+			if ($aArticle['entity'][0] == 'p') {
+				$iPID = (int)substr($aArticle['entity'], 1);
+			} else if ($aArticle['entity'][0] == 'q') {
+				$iQID = (int)substr($aArticle['entity'], 1);
+			} else {
+				continue;
+			}
+
+			echo ".";
+
+			fputcsv($hFileEntity, array($iID,$sTitle,$iPID,$iQID,@$aArticle['datatype']));
+
+			foreach($aArticle['label'] as $sLang => $sLabel) {
+				fputcsv($hFileEntityLabel, array($iID,$sLang,$sLabel));
+				// echo "insert into entity_label values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
+			}
+
+			foreach($aArticle['description'] as $sLang => $sLabel) {
+				fputcsv($hFileEntityDescription, array($iID,$sLang,$sLabel));
+				// echo "insert into entity_description values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
+			}
+
+			foreach($aArticle['aliases'] as $sLang => $aLabels) {
+				$aUniqueAlias = array();
+				foreach($aLabels as $sLabel) {
+					if (!isset($aUniqueAlias[$sLabel]) && $sLabel) {
+					fputcsv($hFileEntityAlias, array($iID,$sLang,$sLabel));
+					// echo "insert into entity_alias values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
+					$aUniqueAlias[$sLabel] = true;
+				}
+			}
+		}
+
+		foreach($aArticle['links'] as $sLang => $sLabel) {
+			fputcsv($hFileEntityLink, array($iID,$sLang,$sLabel));
+			// echo "insert into entity_link values (".$iID.",'".pg_escape_string($sLang)."','".pg_escape_string($sLabel)."');\n";
+		}
+
+
+		if (isset($aArticle['claims'])) {
+
+			foreach($aArticle['claims'] as $iClaim => $aClaim) {
+
+				$bFail = false;
+			if ($aClaim['m'][0] == 'novalue') continue;
+			if ($aClaim['m'][0] == 'somevalue') continue;
+			$iPID = (int)$aClaim['m'][1];
+			if ($aClaim['m'][0] != 'value') $bFail = true;
+			if ($aClaim['m'][2]== 'wikibase-entityid') {
+
+				if ($aClaim['m'][3]['entity-type'] != 'item') $bFail = true;
+				fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,$aClaim['m'][3]['numeric-id'],null,null));
+				// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,".$aClaim['m'][3]['numeric-id'].",null);\n";
+
+			} elseif ($aClaim['m'][2] == 'globecoordinate') {
+
+				if ($aClaim['m'][3]['globe'] != 'http://www.wikidata.org/entity/Q2') $bFail = true;
+				fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,null,"SRID=4326;POINT(".((float)$aClaim['m'][3]['longitude'])." ".((float)$aClaim['m'][3]['latitude']).")",null));
+				// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,null,ST_SetSRID(ST_MakePoint(".((float)$aClaim['m'][3]['longitude']).", ".((float)$aClaim['m'][3]['latitude'])."),4326));\n";
+
+			} elseif ($aClaim['m'][2] == 'time') {
+				// TODO!
+/*
+		if ($aClaim['m'][3]['calendarmodel'] == 'http://www.wikidata.org/entity/Q1985727') {
+			// Gregorian
+			if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
+				if ((int)$aMatch[2] < 4700 && ) {
+					$sDateString = $aMatch[2].'-'.$aMatch[3].($aClaim['m'][3]['timezone']>=0?'+':'').$aClaim['m'][3]['timezone'].($aMatch[1]=='-'?' bc':'');
+					fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,null,null,null,$sDateString));
+				}
+			} else {
+//				$bFail = true;
+			}
+		} elseif ( $aClaim['m'][3]['calendarmodel'] != 'http://www.wikidata.org/entity/Q1985786') {
+/ *
+			// Julian
+			if (preg_match('#(\\+|-)0*([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}:[0-9]{2}:[0-9]{2})Z#', $aClaim['m'][3]['time'], $aMatch)) {
+var_dump($aMatch);
+exit;
+$iDayCount = juliantojd(2, 11, 1732);
+var_dump($iDayCount, jdtogregorian($iDayCount));
+			} else {
+				$bFail = true;
+exit;
+			}
+exit;
+* /
+		} else {
+//			$bFail = true;
+		}
+*/
+			} elseif ($aClaim['m'][2] == 'string') {
+
+				// echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",'".pg_escape_string($aClaim['m'][3])."',null,null);\n";
+				fputcsv($hFileEntityProperty, array($iID,$iClaim,$iPID,$aClaim['m'][3],null,null,null));
+
+			} else {
+
+				$bFail = true;
+
+			}
+
+			// Don't care about sources:	if ($aClaim['refs'] != array()) $bFail = true;
+
+			if ($bFail) {
+				var_dump($sTitle);
+				var_dump($aClaim);
+			} else {
+				// process
+			}
+
+		}
+
+			}
+		}
+	}
+	fclose($hFile);
+	fclose($hFileEntity);
+	fclose($hFileEntityLabel);
+	fclose($hFileEntityDescription);
+	fclose($hFileEntityAlias);
+	fclose($hFileEntityLink);
+	fclose($hFileEntityProperty);
+}

wikidata/pull_wikipedia_logs.php

@@ -0,0 +1,95 @@
+<?php
+
+	for($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, 15, 2013); $iTimestamp += 24*60*60)
+	{
+		$sYear = date("Y", $iTimestamp);
+		$sMonth = date("Y-m", $iTimestamp);
+		$sDay = date("Ymd", $iTimestamp);
+
+		for($iHour = 0; $iHour < 24; $iHour++)
+		{
+			$sFilename = sprintf("pagecounts-".$sDay."-%02d0000", $iHour);
+			echo $sFilename."\n";
+			if (!file_exists($sFilename.'.gz'))
+			{
+				exec('wget http://dumps.wikimedia.org/other/pagecounts-raw/'.$sYear.'/'.$sMonth.'/'.$sFilename.'.gz');
+			}
+
+			exec('gzip -dc '.$sFilename.'.gz'.' | grep -e "^[a-z]\{2\} [^ :]\+ [0-9]\+" > hour.txt');
+
+			$hPrevTotals = @fopen("totals.txt", "r");
+			$hDayTotals = @fopen("hour.txt", "r");
+			$hNewTotals = @fopen("newtotals.txt", "w");
+
+			$sPrevKey = $sDayKey = true;
+			$sPrevLine = true;
+			$sDayLine = true;
+
+			do
+			{
+				if ($sPrevKey === $sDayKey)
+				{
+					if ($sPrevLine !== true) fputs($hNewTotals, "$sPrevKey ".($iPrevValue+$iDayValue)."\n");
+					$sPrevLine = true;
+					$sDayLine = true;
+				}
+				else if ($sDayKey !== false && ($sPrevKey > $sDayKey || $sPrevKey === false))
+				{
+					fputs($hNewTotals, "$sDayKey ".($iDayValue)."\n");
+					$sDayLine = true;
+				}
+				else if ($sPrevKey !== false && ($sDayKey > $sPrevKey || $sDayKey === false))
+				{
+					fputs($hNewTotals, "$sPrevKey ".($iPrevValue)."\n");
+					$sPrevLine = true;
+				}
+
+				if ($sPrevLine === true)
+				{
+					$sPrevLine = $hPrevTotals?fgets($hPrevTotals, 4096):false;
+					if ($sPrevLine !== false)
+					{
+						$aPrevLine = explode(' ', $sPrevLine);
+						$sPrevKey = $aPrevLine[0].' '.$aPrevLine[1];
+						$iPrevValue = (int)$aPrevLine[2];
+					}
+					else
+					{
+						$sPrevKey = false;
+						$iPrevValue =  0;
+					}
+				}
+
+				if ($sDayLine === true)
+				{
+					$sDayLine = $hDayTotals?fgets($hDayTotals, 4096):false;
+					if ($sDayLine !== false)
+					{
+						preg_match('#^([a-z]{2}) ([^ :]+) ([0-9]+) [0-9]+$#', $sDayLine, $aMatch);
+						$sDayKey = $aMatch[1].' '.$aMatch[2];
+						$iDayValue = (int)$aMatch[3];
+					}
+					else
+					{
+						$sDayKey = false;
+						$iDayValue = 0;
+					}
+				}
+
+			} while ($sPrevLine !== false || $sDayLine !== false);
+
+			@fclose($hPrevTotals);
+			@fclose($hDayTotals);
+			@fclose($hNewTotals);
+
+			@unlink("totals.txt");
+			rename("newtotals.txt", "totals.txt");
+		}
+	}
+
+// Notes:
+/*
+ gzip -dc $FILE.gz | grep -e "^en [^ :]\+ [0-9]\+" | sed "s#\(^[a-z]\{2\}\) \([^ :]\+\) \([0-9]\+\) [0-9]\+#update wikipedia_article set hit_count = coalesce(hit_count,0) + \3 where language = '\1' and title = catch_decode_url_part('\2');#g" | /opt/mapquest/stdbase-dev$
+ cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = s,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
+ cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = coalesce(hits,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g"
+*/