Merge branch 'master' of github.com:twain47/Nominatim

AUTHORS

@@ -1 +1,14 @@
-Nominatim was written by Brian Quinion.
+Nominatim was written by:
+
+  Brian Quinion
+  Sarah Hoffmann
+  Frederik Ramm
+  Michael Spreng
+  Daniele Forsi
+  mfn
+  Grant Slater
+  Andree Klattenhoff
+  IrlJidel
+  appelflap
+  b3nn0
+  Spin0us

ChangeLog

@@ -0,0 +1,28 @@
+2.0.1
+
+ * delete outdated entries from location_area_country
+ * remove remaining uses of INTEGER, to allow node ids larger than 2^31
+
+2.1
+
+ * update to openlayers 2.12 (now custom built)
+ * update fallback OSM boundaries
+ * add support for postgresql 9.2/9.3 and postgis 2.x
+ * add structured queries
+ * add options for polygon output in various formats
+   (geojson, svg, kml, postgis text)
+ * maintenance functions for deleting objects and updating regions
+   (plcae_force_update/place_force_delete)
+ * web view for polygons that need deleting
+ * rate limiting using memcache
+ * improve layout of details page
+ * add support for boundary:postal_code
+ * full CORS support
+ * improve parenting of POIs
+ * support for extract daily diffs from Geofabrik
+ * support for addresses without a street
+   (addr:place and conscription number house numbers)
+ * improve layout of word and search_name_* tables
+ * support for US ZIP+4 codes
+ * refactoring of front-end PHP code
+ * lots of smaller bug fixes

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT(Nominatim,2.1)
+AC_INIT(Nominatim,2.2)
 if git rev-parse HEAD 2>/dev/null >/dev/null; then
     AC_SUBST([PACKAGE_VERSION], [$PACKAGE_VERSION-git-`git rev-parse --short HEAD`])
 fi

lib/Geocode.php

@@ -33,6 +33,7 @@
 		protected $iMinAddressRank = 0;
 		protected $iMaxAddressRank = 30;
 		protected $aAddressRankList = array();
+		protected $exactMatchCache = array();
 
 		protected $sAllowedTypesSQLList = false;
 
@@ -277,13 +278,13 @@
 			// Get the details for display (is this a redundant extra step?)
 			$sPlaceIDs = join(',',$aPlaceIDs);
 
-			$sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id,calculated_country_code as country_code,";
+			$sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id, calculated_country_code as country_code,";
 			$sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
 			$sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,";
 			$sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,";
 			$sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 			$sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance, ";
-			$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(placex.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
+			$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(CASE WHEN placex.rank_search < 28 THEN placex.place_id ELSE placex.parent_place_id END) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
 			$sSQL .= "(extratags->'place') as extra_place ";
 			$sSQL .= "from placex where place_id in ($sPlaceIDs) ";
 			$sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank ";
@@ -302,26 +303,26 @@
 			if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank)
 			{
 				$sSQL .= " union ";
-				$sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,";
+				$sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,";
 				$sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
 				$sSQL .= "null as placename,";
 				$sSQL .= "null as ref,";
 				$sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 				$sSQL .= "-0.15 as importance, ";
-				$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
+				$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
 				$sSQL .= "null as extra_place ";
 				$sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) ";
 				$sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
 				$sSQL .= "group by place_id";
 				if (!$this->bDeDupe) $sSQL .= ",place_id";
 				$sSQL .= " union ";
-				$sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,";
+				$sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,";
 				$sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,";
 				$sSQL .= "null as placename,";
 				$sSQL .= "null as ref,";
 				$sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, ";
 				$sSQL .= "-0.10 as importance, ";
-				$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
+				$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
 				$sSQL .= "null as extra_place ";
 				$sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
 				$sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
@@ -342,6 +343,36 @@
 			return $aSearchResults;
 		}
 
+		/* Perform the actual query lookup.
+
+			Returns an ordered list of results, each with the following fields:
+			  osm_type: type of corresponding OSM object
+							N - node
+							W - way
+							R - relation
+							P - postcode (internally computed)
+			  osm_id: id of corresponding OSM object
+			  class: general object class (corresponds to tag key of primary OSM tag)
+			  type: subclass of object (corresponds to tag value of primary OSM tag)
+			  admin_level: see http://wiki.openstreetmap.org/wiki/Admin_level
+			  rank_search: rank in search hierarchy
+							(see also http://wiki.openstreetmap.org/wiki/Nominatim/Development_overview#Country_to_street_level)
+			  rank_address: rank in address hierarchy (determines orer in address)
+			  place_id: internal key (may differ between different instances)
+			  country_code: ISO country code
+			  langaddress: localized full address
+			  placename: localized name of object
+			  ref: content of ref tag (if available)
+			  lon: longitude
+			  lat: latitude
+			  importance: importance of place based on Wikipedia link count
+			  addressimportance: cumulated importance of address elements
+			  extra_place: type of place (for admin boundaries, if there is a place tag)
+			  aBoundingBox: bounding Box
+			  label: short description of the object class/type (English only) 
+			  name: full name (currently the same as langaddress)
+			  foundorder: secondary ordering for places with same importance
+		*/
 		function lookup()
 		{
 			if (!$this->sQuery && !$this->aStructuredQuery) return false;
@@ -1148,12 +1179,16 @@
 							$aOrder[] = "$sImportanceSQL DESC";
 							if (sizeof($aSearch['aFullNameAddress']))
 							{
-								$aOrder[] = '(select count(*) from (select unnest(ARRAY['.join($aSearch['aFullNameAddress'],",").']) INTERSECT select unnest(nameaddress_vector))s) DESC';
+								$sExactMatchSQL = '(select count(*) from (select unnest(ARRAY['.join($aSearch['aFullNameAddress'],",").']) INTERSECT select unnest(nameaddress_vector))s) as exactmatch';
+								$aOrder[] = 'exactmatch DESC';
+							} else {
+								$sExactMatchSQL = '0::int as exactmatch';
 							}
 
 							if (sizeof($aTerms))
 							{
-								$sSQL = "select place_id";
+								$sSQL = "select place_id, ";
+								$sSQL .= $sExactMatchSQL;
 								$sSQL .= " from search_name";
 								$sSQL .= " where ".join(' and ',$aTerms);
 								$sSQL .= " order by ".join(', ',$aOrder);
@@ -1181,6 +1216,7 @@
 									//if ($aViewBoxRow['in_small'] == 't') $bViewBoxMatch = 1;
 									//else if ($aViewBoxRow['in_large'] == 't') $bViewBoxMatch = 2;
 									$aPlaceIDs[] = $aViewBoxRow['place_id'];
+									$this->exactMatchCache[$aViewBoxRow['place_id']] = $aViewBoxRow['exactmatch'];
 								}
 							}
 							//var_Dump($aPlaceIDs);
@@ -1586,7 +1622,24 @@
 				$aResult['importance'] = $aResult['importance'] + ($iCountWords*0.1); // 0.1 is a completely arbitrary number but something in the range 0.1 to 0.5 would seem right
 
 				$aResult['name'] = $aResult['langaddress'];
-				$aResult['foundorder'] = -$aResult['addressimportance'];
+				// secondary ordering (for results with same importance (the smaller the better):
+				//   - approximate importance of address parts
+				$aResult['foundorder'] = -$aResult['addressimportance']/10;
+				//   - number of exact matches from the query
+				if (isset($this->exactMatchCache[$aResult['place_id']]))
+					$aResult['foundorder'] -= $this->exactMatchCache[$aResult['place_id']];
+				else if (isset($this->exactMatchCache[$aResult['parent_place_id']]))
+					$aResult['foundorder'] -= $this->exactMatchCache[$aResult['parent_place_id']];
+				//  - importance of the class/type
+				if (isset($aClassType[$aResult['class'].':'.$aResult['type']]['importance'])
+					&& $aClassType[$aResult['class'].':'.$aResult['type']]['importance'])
+				{
+					$aResult['foundorder'] = $aResult['foundorder'] + 0.000001 * $aClassType[$aResult['class'].':'.$aResult['type']]['importance'];
+				}
+				else
+				{
+					$aResult['foundorder'] = $aResult['foundorder'] + 0.001;
+				}
 				$aSearchResults[$iResNum] = $aResult;
 			}
 			uasort($aSearchResults, 'byImportance');

lib/ReverseGeocode.php

@@ -95,8 +95,8 @@
 				$sSQL .= ' and rank_search != 28 and rank_search >= '.$iMaxRank;
 				$sSQL .= ' and (name is not null or housenumber is not null)';
 				$sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\')';
-				$sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') ';
 				$sSQL .= ' and indexed_status = 0 ';
+				$sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') ';
 				$sSQL .= ' OR ST_DWithin('.$sPointSQL.', centroid, '.$fSearchDiam.'))';
 				$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1';
 				if (CONST_Debug) var_dump($sSQL);

lib/init-website.php

@@ -55,8 +55,7 @@
 	if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit)
 	{
 		echo "Your IP has been blocked. \n";
-		echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n";
+		echo CONST_BlockMessage;
-		echo "Information on the Nominatim usage policy can be found here: http://wiki.openstreetmap.org/wiki/Nominatim#Usage_Policy \n";
 		exit;
 	}
 

lib/template/details-html.php

@@ -80,7 +80,7 @@
 	echo '<h1>';
 	if ($aPointDetails['icon'])
 	{
-		echo '<img style="float:right;margin-right:40px;" src="'.CONST_Website_BaseURL.'images/mapicons/'.$aPointDetails['icon'].'.n.32.png'.'">';
+		echo '<img style="float:right;margin-right:40px;" src="'.CONST_Website_BaseURL.'images/mapicons/'.$aPointDetails['icon'].'.n.32.png'.'" alt="'.$aPointDetails['icon'].'" />';
 	}
 	echo $aPointDetails['localname'].'</h1>';
 	echo '<div class="locationdetails">';

lib/template/search-xml.php

@@ -80,7 +80,7 @@
 		echo " display_name='".htmlspecialchars($aResult['name'], ENT_QUOTES)."'";
 
 		echo " class='".htmlspecialchars($aResult['class'])."'";
-		echo " type='".htmlspecialchars($aResult['type'])."'";
+		echo " type='".htmlspecialchars($aResult['type'], ENT_QUOTES)."'";
 		echo " importance='".htmlspecialchars($aResult['importance'])."'";
 		if (isset($aResult['icon']) && $aResult['icon'])
 		{

munin/nominatim_importlag

@@ -5,26 +5,42 @@
 # Can be configured through libpq environment variables, for example
 # PGUSER, PGDATABASE, etc. See man page of psql for more information
 #
-# Parameters: 
+# To configure munin for a default installation, add these lines to
+# the file /etc/munin/plugin-conf.d/munin-node or in any file in the
+# directory /etc/munin/plugin-conf.d/
+#
+# [nominatim_*]
+# user www-data
+# env.PGUSER www-data
+# env.PGPORT 5432
+# env.PGDATABASE nominatim
+# env.age_warning 21600
+# env.age_critical 86400
+
+# Parameters:
 #
 #       config   (required)
 #       autoconf (optional - used by munin-config)
 #
- 
+
+. $MUNIN_LIBDIR/plugins/plugin.sh
+
 if [ "$1" = "config" ]; then
- 
+
-        echo 'graph_title Data import lag' 
+        echo 'graph_title Data import lag'
-        echo 'graph_args --base 1000 -l 0' 
+        echo 'graph_args --base 1000 -l 0'
-        echo 'graph_vlabel minutes' 
+        echo 'graph_vlabel minutes'
-        echo 'graph_category nominatim' 
+        echo 'graph_category nominatim'
-        echo 'age.label DB import age' 
+        echo 'age.label DB import age'
-        echo 'age.type GAUGE' 
+        echo 'age.type GAUGE'
         echo 'age.cdef age,60,/'
+        print_warning age
+        print_critical age
         exit 0
 fi
- 
+
 
 delay=`psql -c "copy (select extract(epoch from timezone('utc', now())-lastimportdate)::int from import_status) to stdout"`
- 
+
- 
+
 echo "age.value $delay"

nominatim/nominatim.c

@@ -115,7 +115,7 @@ int main(int argc, char *argv[])
 
     PGconn *conn;
 
-    fprintf(stderr, "nominatim SVN version %s\n\n", VERSION);
+    fprintf(stderr, "nominatim version %s\n\n", VERSION);
 
     while (1)
     {

settings/settings.php

@@ -55,6 +55,7 @@
 	@define('CONST_ClosedForIndexingExceptionIPs', '');
 	@define('CONST_BlockedIPs', '');
 	@define('CONST_BulkUserIPs', '');
+	@define('CONST_BlockMessage', ''); // additional info to show for blocked IPs
 
 	@define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/');
 	@define('CONST_Tile_Default', 'Mapnik');

sql/functions.sql

@@ -347,7 +347,7 @@ BEGIN
 
     w := getorcreate_word_id(s);
 
-    IF NOT (ARRAY[w] <@ result) THEN
+    IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
       result := result || w;
     END IF;
 
@@ -415,7 +415,7 @@ BEGIN
 
   w := getorcreate_word_id(s);
 
-  IF NOT (ARRAY[w] <@ result) THEN
+  IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN
     result := result || w;
   END IF;
 
@@ -938,10 +938,6 @@ BEGIN
   --DEBUG: RAISE WARNING '% %',NEW.osm_type,NEW.osm_id;
 
   -- just block these
-  IF NEW.class = 'highway' and NEW.type in ('turning_circle','traffic_signals','mini_roundabout','noexit','crossing') THEN
---    RAISE WARNING 'bad highway %',NEW.osm_id;
-    RETURN null;
-  END IF;
   IF NEW.class in ('landuse','natural') and NEW.name is null THEN
 --    RAISE WARNING 'empty landuse %',NEW.osm_id;
     RETURN null;
@@ -1127,7 +1123,11 @@ BEGIN
         return NULL;
       END IF;
       NEW.rank_search := NEW.admin_level * 2;
-      NEW.rank_address := NEW.rank_search;
+      IF NEW.type = 'administrative' THEN
+        NEW.rank_address := NEW.rank_search;
+      ELSE
+        NEW.rank_address := 0;
+      END IF;
     ELSEIF NEW.class = 'landuse' AND ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') THEN
       NEW.rank_search := 22;
       NEW.rank_address := NEW.rank_search;
@@ -1245,7 +1245,7 @@ BEGIN
    -- Note: won't work on initial import because the classtype tables
    -- do not yet exist. It won't hurt either.
   classtable := 'place_classtype_' || NEW.class || '_' || NEW.type;
-  SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable INTO result;
+  SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable and schemaname = current_schema() INTO result;
   IF result THEN
     EXECUTE 'INSERT INTO ' || classtable::regclass || ' (place_id, centroid) VALUES ($1,$2)' 
     USING NEW.place_id, ST_Centroid(NEW.geometry);
@@ -1987,7 +1987,7 @@ BEGIN
 
   -- remove from tables for special search
   classtable := 'place_classtype_' || OLD.class || '_' || OLD.type;
-  SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable INTO b;
+  SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable and schemaname = current_schema() INTO b;
   IF b THEN
     EXECUTE 'DELETE FROM ' || classtable::regclass || ' WHERE place_id = $1' USING OLD.place_id;
   END IF;
@@ -2044,9 +2044,6 @@ BEGIN
   END IF;
 
   -- Just block these - lots and pointless
-  IF NEW.class = 'highway' and NEW.type in ('turning_circle','traffic_signals','mini_roundabout','noexit','crossing') THEN
-    RETURN null;
-  END IF;
   IF NEW.class in ('landuse','natural') and NEW.name is null THEN
     RETURN null;
   END IF;
@@ -2097,6 +2094,9 @@ BEGIN
   IF existingplacex.osm_type IS NULL THEN
 
     IF existing.osm_type IS NOT NULL THEN
+      -- pathological case caused by the triggerless copy into place during initial import
+      -- force delete even for large areas, it will be reinserted later
+      UPDATE place set geometry = ST_SetSRID(ST_Point(0,0), 4326) where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
       DELETE from place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type;
     END IF;
 
@@ -2255,7 +2255,7 @@ BEGIN
       country_code = NEW.country_code,
       parent_place_id = null,
       extratags = NEW.extratags,
-      admin_level = NEW.admin_level,
+      admin_level = CASE WHEN NEW.admin_level > 15 THEN 15 ELSE NEW.admin_level END,
       indexed_status = 2,    
       geometry = NEW.geometry
       where place_id = existingplacex.place_id;
@@ -2481,7 +2481,7 @@ BEGIN
       CASE WHEN class = 'place' and type = 'postcode' THEN hstore('name', postcode) ELSE name END as name,
       class, type, admin_level, fromarea, isaddress,
       CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address,
-      distance,calculated_country_code
+      distance,calculated_country_code,postcode
       from place_addressline join placex on (address_place_id = placex.place_id) 
       where place_addressline.place_id = for_place_id 
       and (cached_rank_address > 0 AND cached_rank_address < searchrankaddress)
@@ -2496,6 +2496,9 @@ BEGIN
     IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN
       location.isaddress := FALSE;
     END IF;
+    IF searchpostcode IS NULL and location.isaddress and location.type != 'postcode' and location.postcode IS NOT NULL THEN
+      searchpostcode := location.postcode;
+    END IF;
     IF location.rank_address = 4 AND location.isaddress THEN
       hadcountry := true;
     END IF;

sql/tables.sql

@@ -212,16 +212,9 @@ CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name
 DROP SEQUENCE seq_place;
 CREATE SEQUENCE seq_place start 1;
 GRANT SELECT on placex to "www-data" ;
-GRANT UPDATE ON placex to "www-data" ;
 GRANT SELECT ON search_name to "www-data" ;
-GRANT DELETE on search_name to "www-data" ;
-GRANT INSERT on search_name to "www-data" ;
 GRANT SELECT on place_addressline to "www-data" ;
-GRANT INSERT ON place_addressline to "www-data" ;
-GRANT DELETE on place_addressline to "www-data" ;
 GRANT SELECT ON seq_word to "www-data" ;
-GRANT UPDATE ON seq_word to "www-data" ;
-GRANT INSERT ON word to "www-data" ;
 GRANT SELECT ON planet_osm_ways to "www-data" ;
 GRANT SELECT ON planet_osm_rels to "www-data" ;
 GRANT SELECT on location_area to "www-data" ;

utils/setup.php

@@ -125,7 +125,13 @@
 			pgsqlRunScript('CREATE EXTENSION hstore');
 		}
 
-		pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
+		$fPostgisVersion = (float) CONST_Postgis_Version;
+		if ($fPostgisVersion < 2.0) {
+			pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
+			pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
+		} else {
+			pgsqlRunScript('CREATE EXTENSION postgis');
+		}
 		$sVersionString = $oDB->getOne('select postgis_full_version()');
 		preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
 		if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
@@ -134,7 +140,6 @@
 			exit;
 		}
 
-		pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');

utils/tigerAddressImport.py

@@ -3325,6 +3325,102 @@ county_fips = {
 '56041' : 'Uinta, WY' ,
 '56043' : 'Washakie, WY' ,
 '56045' : 'Weston, WY' ,
+'60000' : 'AMERICAN SAMOA',
+'60010' : 'Eastern district, AS' ,
+'60020' : 'Manua district, AS' ,
+'60030' : 'Rose island, AS' ,
+'60040' : 'Swains island, AS' ,
+'60050' : 'Western district, AS' ,
+'66000' : 'GUAM' ,
+'66010' : 'Guam, GU' ,
+'69000' : 'COMMONWEALTH OF THE NORTHERN MARIANA ISLANDS' ,
+'69085' : 'Commonwealth of the Northern Mariana Islands, MP' ,
+'69100' : 'Commonwealth of the Northern Mariana Islands, MP' ,
+'69110' : 'Commonwealth of the Northern Mariana Islands, MP' ,
+'69120' : 'Commonwealth of the Northern Mariana Islands, MP' ,
+'72000' : 'PUERTO RICO', 
+'72001' : 'Adjuntas, PR', 
+'72003' : 'Aguada, PR', 
+'72005' : 'Aguadilla, PR', 
+'72007' : 'Aguas Buenas, PR', 
+'72009' : 'Aibonito, PR', 
+'72011' : 'Anasco, PR', 
+'72013' : 'Arecibo, PR', 
+'72015' : 'Arroyo, PR', 
+'72017' : 'Barceloneta, PR', 
+'72019' : 'Barranquitas, PR', 
+'72021' : 'Bayamon, PR', 
+'72023' : 'Cabo Rojo, PR', 
+'72025' : 'Caguas, PR', 
+'72027' : 'Camuy, PR', 
+'72029' : 'Canovanas, PR', 
+'72031' : 'Carolina, PR', 
+'72033' : 'Catano, PR', 
+'72035' : 'Cayey, PR', 
+'72037' : 'Ceiba, PR', 
+'72039' : 'Ciales, PR', 
+'72041' : 'Cidra, PR', 
+'72043' : 'Coamo, PR', 
+'72045' : 'Comerio, PR', 
+'72047' : 'Corozal, PR', 
+'72049' : 'Culebra, PR', 
+'72051' : 'Dorado, PR', 
+'72053' : 'Fajardo, PR', 
+'72054' : 'Florida, PR', 
+'72055' : 'Guanica, PR', 
+'72057' : 'Guayama, PR', 
+'72059' : 'Guayanilla, PR', 
+'72061' : 'Guaynabo, PR', 
+'72063' : 'Gurabo, PR', 
+'72065' : 'Hatillo, PR', 
+'72067' : 'Hormigueros, PR', 
+'72069' : 'Humacao, PR', 
+'72071' : 'Isabela, PR', 
+'72073' : 'Jayuya, PR', 
+'72075' : 'Juana Diaz, PR', 
+'72077' : 'Juncos, PR', 
+'72079' : 'Lajas, PR', 
+'72081' : 'Lares, PR', 
+'72083' : 'Las Marias, PR', 
+'72085' : 'Las Piedras, PR', 
+'72087' : 'Loiza, PR', 
+'72089' : 'Luquillo, PR', 
+'72091' : 'Manati, PR', 
+'72093' : 'Maricao, PR', 
+'72095' : 'Maunabo, PR', 
+'72097' : 'Mayaguez, PR', 
+'72099' : 'Moca, PR', 
+'72101' : 'Morovis, PR', 
+'72103' : 'Naguabo, PR', 
+'72105' : 'Naranjito, PR', 
+'72107' : 'Orocovis, PR', 
+'72109' : 'Patillas, PR', 
+'72111' : 'Penuelas, PR', 
+'72113' : 'Ponce, PR', 
+'72115' : 'Quebradillas, PR', 
+'72117' : 'Rincon, PR', 
+'72119' : 'Rio Grande, PR', 
+'72121' : 'Sabana Grande, PR', 
+'72123' : 'Salinas, PR', 
+'72125' : 'San German, PR', 
+'72127' : 'San Juan, PR', 
+'72129' : 'San Lorenzo, PR', 
+'72131' : 'San Sebastian, PR', 
+'72133' : 'Santa Isabel, PR', 
+'72135' : 'Toa Alta, PR', 
+'72137' : 'Toa Baja, PR', 
+'72139' : 'Trujillo Alto, PR', 
+'72141' : 'Utuado, PR', 
+'72143' : 'Vega Alta, PR', 
+'72145' : 'Vega Baja, PR', 
+'72147' : 'Vieques, PR', 
+'72149' : 'Villalba, PR', 
+'72151' : 'Yabucoa, PR', 
+'72153' : 'Yauco, PR', 
+'78000' : 'VIRGIN ISLANDS' ,
+'78010' : 'St. Croix, VI' ,
+'78020' : 'St. John, VI' ,
+'78030' : 'St. Thomas, VI' ,
 }
 
 def fipsstate(fips,countyfp):

utils/update.php

@@ -74,6 +74,7 @@
 	$oDB =& getDB();
 
 	$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
+	if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
 
 	// cache memory to be used by osm2pgsql, should not be more than the available memory
 	$iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000);
@@ -82,7 +83,7 @@
 		$iCacheMemory = getCacheMemoryMB();
 		echo "WARNING: resetting cache memory to $iCacheMemory\n";
 	}
-	$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'];
+	$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'];
 	if (!is_null(CONST_Osm2pgsql_Flatnode_File))
 	{
 		$sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
@@ -360,7 +361,7 @@
 
 	if ($aResult['index'])
 	{
-		passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
+		passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']);
 	}
 
 	if ($aResult['import-osmosis'] || $aResult['import-osmosis-all'])
@@ -377,7 +378,7 @@
 		$sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
 		$sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
 		$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
-		$sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'];
+		$sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
 		if (!$aResult['no-npi']) {
 			$sCMDIndex .= '-F ';
 		}
@@ -417,8 +418,9 @@
 							exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); 
 						}
 						// There are new replication files - use osmosis to download the file
-						echo "\nReplication Delay is ".$aReplicationLag[0]."\n";
+						echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
 					}
+					$fStartTime = time();
 					$fCMDStartTime = time();
 					echo $sCMDDownload."\n";
 					exec($sCMDDownload, $sJunk, $iErrorLevel);
@@ -431,9 +433,10 @@
 					}
 					$iFileSize = filesize($sImportFile);
 					$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
-					echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 					$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
+					var_Dump($sSQL);
 					$oDB->query($sSQL);
+					echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 				}
 
 				$iFileSize = filesize($sImportFile);
@@ -446,12 +449,12 @@
 				if ($iErrorLevel)
 				{
 					echo "Error: $iErrorLevel\n";
-					exit;
+					exit($iErrorLevel);
 				}
-				echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 				$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
 				var_Dump($sSQL);
 				$oDB->query($sSQL);
+				echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 
 				// Archive for debug?
 				unlink($sImportFile);
@@ -461,15 +464,15 @@
 
 			// Index file
 			$sThisIndexCmd = $sCMDIndex;
+			$fCMDStartTime = time();
 
 			if (!$aResult['no-npi'])
 			{
-				$fCMDStartTime = time();
 				$iFileID = $oDB->getOne('select nextval(\'file\')');
 				if (PEAR::isError($iFileID))
 				{
 					echo $iFileID->getMessage()."\n";
-					exit;
+					exit(-1);
 				} 
 				$sFileDir = CONST_BasePath.'/export/diff/';
 				$sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT);
@@ -495,7 +498,7 @@
 				if ($iErrorLevel)
 				{
 					echo "Error: $iErrorLevel\n";
-					exit;
+					exit($iErrorLevel);
 				}
 
 				if (!$aResult['no-npi'])
@@ -510,7 +513,7 @@
 					if ($iErrorLevel)
 					{
 						echo "Error: $iErrorLevel\n";
-						exit;
+						exit($iErrorLevel);
 					}
 
 					rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2",
@@ -518,26 +521,27 @@
 				}
 			}
 
-			echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 			$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
+			var_Dump($sSQL);
 			$oDB->query($sSQL);
+			echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
 
 			$sSQL = "update import_status set lastimportdate = '$sBatchEnd'";
 			$oDB->query($sSQL);
 
 			$fDuration = time() - $fStartTime;
-			echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n";
+			echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n";
 			if (!$aResult['import-osmosis-all']) exit;
 
 			if ( CONST_Replication_Update_Interval > 60 )
 			{
-				$iSleep = round(CONST_Replication_Update_Interval*0.8);
+				$iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time()));
 			}
 			else
 			{
 				$iSleep = max(0,CONST_Replication_Update_Interval-$fDuration);
 			}
-			echo "Sleeping $iSleep seconds\n";
+			echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
 			sleep($iSleep);
 		}