make tables for external data (Tiger and aux) configurable

Introduces two new settings CONST_Use_US_Tiger_Data and
CONST_Use_Aux_Location_data, which are disabled by default.
When false the corresponding tables are ignored in queries
and updates.

Aux and tiger tables are no longer created by default. This
has to be done by the corresponding import scripts. The former
aux table creation can be found in sql/aux_tables.sql for
reference.

lib/Geocode.php

@@ -431,56 +431,62 @@
 
 			if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank)
 			{
-				//query also location_property_tiger and location_property_aux
+				if (CONST_Use_US_Tiger_Data)
-				//Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
+				{
-				//only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
+					//query also location_property_tiger and location_property_aux
-				$sHousenumbers = "";
+					//Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
-				$i = 0;
+					//only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
-				$length = count($aPlaceIDs);
+					$sHousenumbers = "";
-				foreach($aPlaceIDs as $placeID => $housenumber)
+					$i = 0;
-                {
+					$length = count($aPlaceIDs);
-					$i++;
+					foreach($aPlaceIDs as $placeID => $housenumber)
-					$sHousenumbers .= "(".$placeID.", ".$housenumber.")";
+					{
-					if($i<$length)
+						$i++;
-						$sHousenumbers .= ", ";
+						$sHousenumbers .= "(".$placeID.", ".$housenumber.")";
+						if($i<$length)
+							$sHousenumbers .= ", ";
+					}
+
+					$sSQL .= "union ";
+					$sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
+					$sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
+					$sSQL .= ", null as placename";
+					$sSQL .= ", null as ref";
+					if ($this->bIncludeExtraTags) $sSQL .= ", null as extra";
+					if ($this->bIncludeNameDetails) $sSQL .= ", null as names";
+					$sSQL .= ", avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
+					$sSQL .= $sImportanceSQL."-1.15 as importance ";
+					$sSQL .= ", (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";
+					$sSQL .= ", null as extra_place ";
+					$sSQL .= " from (select place_id";
+					//interpolate the Tiger housenumbers here
+					$sSQL .= ", ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) as centroid, parent_place_id, housenumber_for_place ";
+					$sSQL .= "from (location_property_tiger ";
+					$sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
+					$sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
+					$sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
+					if (!$this->bDeDupe) $sSQL .= ", place_id ";
 				}
 
-				$sSQL .= "union ";
+				if (CONST_Use_Aux_Location_data)
-				$sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
+				{
-				$sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
+					$sSQL .= " union ";
-				$sSQL .= ", null as placename";
+					$sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
-				$sSQL .= ", null as ref";
+					$sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
-				if ($this->bIncludeExtraTags) $sSQL .= ", null as extra";
+					$sSQL .= "null as placename, ";
-				if ($this->bIncludeNameDetails) $sSQL .= ", null as names";
+					$sSQL .= "null as ref, ";
-				$sSQL .= ", avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
+					if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
-				$sSQL .= $sImportanceSQL."-1.15 as importance ";
+					if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
-				$sSQL .= ", (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";
+					$sSQL .= "avg(ST_X(centroid)) as lon, avg(ST_Y(centroid)) as lat, ";
-				$sSQL .= ", null as extra_place ";
+					$sSQL .= $sImportanceSQL."-1.10 as importance, ";
-				$sSQL .= " from (select place_id";
+					$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
-				//interpolate the Tiger housenumbers here
+					$sSQL .= "null as extra_place ";
-				$sSQL .= ", ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) as centroid, parent_place_id, housenumber_for_place ";
+					$sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
-				$sSQL .= "from (location_property_tiger ";
+					$sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
-				$sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
+					$sSQL .= "group by place_id";
-				$sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
+					if (!$this->bDeDupe) $sSQL .= ", place_id";
-				$sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
+					$sSQL .= ", get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) ";
-				if (!$this->bDeDupe) $sSQL .= ", place_id ";
+				}
-
-				$sSQL .= " union ";
-				$sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
-				$sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
-				$sSQL .= "null as placename, ";
-				$sSQL .= "null as ref, ";
-				if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
-				if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
-				$sSQL .= "avg(ST_X(centroid)) as lon, avg(ST_Y(centroid)) as lat, ";
-				$sSQL .= $sImportanceSQL."-1.10 as importance, ";
-				$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
-				$sSQL .= "null as extra_place ";
-				$sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
-				$sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
-				$sSQL .= "group by place_id";
-				if (!$this->bDeDupe) $sSQL .= ", place_id";
-				$sSQL .= ", get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) ";
 			}
 
 			$sSQL .= " order by importance desc";
@@ -1431,7 +1437,7 @@
 								$aPlaceIDs = $this->oDB->getCol($sSQL);
 
 								// If nothing found try the aux fallback table
-								if (!sizeof($aPlaceIDs))
+								if (CONST_Use_Aux_Location_data && !sizeof($aPlaceIDs))
 								{
 									$sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
 									if (sizeof($this->aExcludePlaceIDs))
@@ -1444,7 +1450,7 @@
 								}
 								//if nothing was found in placex or location_property_aux, then search in Tiger data for this housenumber(location_property_tiger)
 								$searchedHousenumber = intval($aSearch['sHouseNumber']);
-								if (!sizeof($aPlaceIDs))
+								if (CONST_Use_US_Tiger_Data && !sizeof($aPlaceIDs))
 								{
 									//new query for lines, not housenumbers anymore
 									if($searchedHousenumber%2 == 0){
@@ -1616,9 +1622,12 @@
 						$sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank ";
 						if (14 >= $this->iMinAddressRank && 14 <= $this->iMaxAddressRank) $sSQL .= " OR (extratags->'place') = 'city'";
 						if ($this->aAddressRankList) $sSQL .= " OR placex.rank_address in (".join(',',$this->aAddressRankList).")";
-						$sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
+						if (CONST_Use_US_Tiger_Data)
-						$sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
+						{
-						if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
+							$sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
+							$sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
+							if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
+						}
 						$sSQL .= ")";
 						if (CONST_Debug) var_dump($sSQL);
 						$aFilteredPlaceIDs = $this->oDB->getCol($sSQL);

lib/PlaceLookup.php

@@ -122,7 +122,7 @@
 
 			$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
 
-			if ($this->sType == 'tiger')
+			if (CONST_Use_US_Tiger_Data && $this->sType == 'tiger')
 			{
 				$sSQL = "select place_id,partition, 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, housenumber, null as street, null as isin, postcode,";
 				$sSQL .= " 'us' as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
@@ -166,7 +166,7 @@
 
 			if ($this->bAddressDetails)
 			{
-				if($this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed
+				if(CONST_Use_US_Tiger_Data && $this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed
 					$aAddress = $this->getAddressNames($aPlace['housenumber']);
 				else
 					$aAddress = $this->getAddressNames();

lib/ReverseGeocode.php

@@ -158,7 +158,7 @@
 			}
 
 			// Only street found? If it's in the US we can check TIGER data for nearest housenumber
-			if ($bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
+			if (CONST_Use_US_Tiger_Data && $bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
 			{
 				$fSearchDiam = 0.001;
 				$sSQL = 'SELECT place_id,parent_place_id,30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction';

settings/settings.php

@@ -13,6 +13,16 @@
 	@define('CONST_Limit_Reindexing', true);
 	// Set to false to avoid importing extra postcodes for the US.
 	@define('CONST_Use_Extra_US_Postcodes', true);
+	// Set to true after importing Tiger house number data for the US.
+	// Note: The tables must already exist or queries will throw errors.
+	//       After changing this setting run ./utils/setup --create-functions
+	//       again.
+	@define('CONST_Use_US_Tiger_Data', false);
+	// Set to true after importing other external house number data.
+	// Note: the aux tables must already exist or queries will throw errors.
+	//       After changing this setting run ./utils/setup --create-functions
+	//       again.
+	@define('CONST_Use_Aux_Location_data', false);
 
 	// Proxy settings
 	@define('CONST_HTTP_Proxy', false);

sql/aux_tables.sql

@@ -0,0 +1,6 @@
+CREATE TABLE location_property_aux () INHERITS (location_property);
+CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
+CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
+CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
+GRANT SELECT ON location_property_aux TO "{www-user}";
+

sql/functions.sql

@@ -1024,7 +1024,7 @@ BEGIN
 
   --DEBUG: RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
 
-  RETURN NEW; -- @DIFFUPDATES@ The following is not needed until doing diff updates, and slows the main index process down
+  RETURN NEW; -- %DIFFUPDATES% The following is not needed until doing diff updates, and slows the main index process down
 
   IF NEW.rank_address > 0 THEN
     IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
@@ -1650,6 +1650,7 @@ BEGIN
       END IF;
     END IF;
 
+    -- %NOTIGERDATA% IF 0 THEN
     -- for the USA we have an additional address table.  Merge in zip codes from there too
     IF NEW.rank_search = 26 AND NEW.calculated_country_code = 'us' THEN
       FOR location IN SELECT distinct postcode from location_property_tiger where parent_place_id = NEW.place_id LOOP
@@ -1662,6 +1663,7 @@ BEGIN
         nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
       END LOOP;
     END IF;
+    -- %NOTIGERDATA% END IF;
 
 -- RAISE WARNING 'ISIN: %', isin_tokens;
 
@@ -2257,18 +2259,22 @@ DECLARE
   hadcountry BOOLEAN;
 BEGIN
     --first query tiger data
+  -- %NOTIGERDATA% IF 0 THEN
   select parent_place_id,'us', 30, postcode, null, 'place', 'house' from location_property_tiger 
     WHERE place_id = in_place_id AND in_housenumber>=startnumber AND in_housenumber <= endnumber
     INTO for_place_id,searchcountrycode, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
   IF for_place_id IS NOT NULL THEN
     searchhousenumber = in_housenumber::text;
   END IF;
-  
+  -- %NOTIGERDATA% END IF;
+
+  -- %NOAUXDATA% IF 0 THEN
   IF for_place_id IS NULL THEN
     select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux
       WHERE place_id = in_place_id 
       INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
   END IF;
+  -- %NOAUXDATA% END IF;
 
   IF for_place_id IS NULL THEN
     select parent_place_id, calculated_country_code, housenumber, rank_search, postcode, name, class, type from placex 

sql/tables.sql

@@ -100,15 +100,6 @@ CREATE TABLE location_property (
   );
 SELECT AddGeometryColumn('location_property', 'centroid', 4326, 'POINT', 2);
 
-CREATE TABLE location_property_aux () INHERITS (location_property);
-CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
-CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
-CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
-GRANT SELECT ON location_property_aux TO "{www-user}";
-
-CREATE TABLE location_property_tiger (linegeo GEOMETRY, place_id BIGINT, partition INTEGER, parent_place_id BIGINT, startnumber INTEGER, endnumber INTEGER, interpolationtype TEXT, postcode TEXT);
-GRANT SELECT ON location_property_tiger TO "{www-user}";
-
 drop table IF EXISTS search_name;
 CREATE TABLE search_name (
   place_id BIGINT,

utils/setup.php

@@ -219,9 +219,26 @@
 		if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built");
 		$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
 		$sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate);
-		if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
+		if ($aCMDResult['enable-diff-updates'])
-		if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
+		{
-		if (CONST_Limit_Reindexing) $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
+			$sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
+		}
+		if ($aCMDResult['enable-debug-statements'])
+		{
+			$sTemplate = str_replace('--DEBUG:', '', $sTemplate);
+		}
+		if (CONST_Limit_Reindexing)
+		{
+			$sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
+		}
+		if (!CONST_Use_US_Tiger_Data)
+		{
+			$sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate);
+		}
+		if (!CONST_Use_Aux_Location_data)
+		{
+			$sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate);
+		}
 		pgsqlRunScript($sTemplate);
 	}
 

website/details.php

@@ -56,10 +56,17 @@
 
 	$iPlaceID = (int)$_GET['place_id'];
 
-	$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
+	if (CONST_Use_US_Tiger_Data)
-	if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+	{
-	$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
+		$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
-	if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+		if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+	}
+
+	if (CONST_Use_Aux_Location_data)
+	{
+		$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
+		if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
+	}
 
 	$hLog = logStart($oDB, 'details', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
 

website/hierarchy.php

@@ -47,13 +47,13 @@
 
 	$iPlaceID = (int)$_GET['place_id'];
 
-	$sAuxHouseNumber = false;
+	if (CONST_Use_US_Tiger_Data)
-	$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
-	if ($iParentPlaceID)
 	{
-		$iPlaceID = $iParentPlaceID;
+		$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
+		if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
 	}
-	else
+
+	if (CONST_Use_Aux_Location_data)
 	{
 		$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
 		if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;