more bigint fixes

Makes it easier to patch partition functions on an already existing
database.

AUTHORS

@@ -1 +1,10 @@
-Nominatim was written by Brian Quinion.
+Nominatim was written by:
+
+  Brian Quinion
+  Sarah Hoffmann
+  Frederik Ramm
+  Michael Spreng
+  Daniele Forsi
+  mfn
+  Grant
+  Andree Klattenhoff

ChangeLog

@@ -0,0 +1,4 @@
+2.0.1
+
+ * delete outdated entries from location_area_country
+ * remove remaining uses of INTEGER, to allow node ids larger than 2^31

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT(Nominatim,1.9)
+AC_INIT(Nominatim,2.0)
 if git rev-parse HEAD 2>/dev/null >/dev/null; then
     AC_SUBST([PACKAGE_VERSION], [$PACKAGE_VERSION-git-`git rev-parse --short HEAD`])
 fi

sql/functions.sql

@@ -624,7 +624,7 @@ BEGIN
 
 --  RAISE WARNING 'Adding location with rank > 25 (% rank %)', place_id, rank_search;
 
-  x := deleteLocationArea(partition, place_id);
+  x := deleteLocationArea(partition, place_id, rank_search);
 
   isarea := false;
   IF (ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(geometry)) THEN
@@ -691,7 +691,7 @@ CREATE OR REPLACE FUNCTION update_location(
 DECLARE
   b BOOLEAN;
 BEGIN
-  b := deleteLocationArea(partition, place_id);
+  b := deleteLocationArea(partition, place_id, rank_search);
 --  result := add_location(NEW.place_id, NEW.country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, NEW.geometry);
   RETURN add_location(place_id, place_country_code, name, rank_search, rank_address, geometry);
 END;
@@ -769,8 +769,8 @@ CREATE OR REPLACE FUNCTION create_interpolation(wayid BIGINT, interpolationtype
 DECLARE
   
   newpoints INTEGER;
-  waynodes integer[];
-  nodeid INTEGER;
+  waynodes BIGINT[];
+  nodeid BIGINT;
   prevnode RECORD;
   nextnode RECORD;
   startnumber INTEGER;
@@ -798,10 +798,10 @@ BEGIN
 
       FOR nodeidpos in 1..array_upper(waynodes, 1) LOOP
 
-        select min(place_id) from placex where osm_type = 'N' and osm_id = waynodes[nodeidpos]::INTEGER and type = 'house' INTO search_place_id;
+        select min(place_id) from placex where osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT and type = 'house' INTO search_place_id;
         IF search_place_id IS NULL THEN
           -- null record of right type
-          select * from placex where osm_type = 'N' and osm_id = waynodes[nodeidpos]::INTEGER and type = 'house' limit 1 INTO nextnode;
+          select * from placex where osm_type = 'N' and osm_id = waynodes[nodeidpos]::BIGINT and type = 'house' limit 1 INTO nextnode;
           select ST_SetSRID(ST_Point(lon::float/10000000,lat::float/10000000),4326) from planet_osm_nodes where id = waynodes[nodeidpos] INTO nextnode.geometry;
           IF nextnode.geometry IS NULL THEN
             -- we don't have any information about this point, most likely
@@ -1304,7 +1304,7 @@ BEGIN
       DELETE FROM place_addressline WHERE place_id = NEW.place_id;
       DELETE FROM place_boundingbox where place_id = NEW.place_id;
       result := deleteRoad(NEW.partition, NEW.place_id);
-      result := deleteLocationArea(NEW.partition, NEW.place_id);
+      result := deleteLocationArea(NEW.partition, NEW.place_id, NEW.rank_search);
       UPDATE placex set linked_place_id = null where linked_place_id = NEW.place_id;
     END IF;
 
@@ -1384,7 +1384,7 @@ BEGIN
             FOR i IN 1..array_upper(relation.members, 1) BY 2 LOOP
               IF NEW.parent_place_id IS NULL AND relation.members[i+1] = 'street' THEN
 --RAISE WARNING 'node in relation %',relation;
-                SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::integer 
+                SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::bigint 
                   and rank_search = 26 INTO NEW.parent_place_id;
               END IF;
             END LOOP;
@@ -1413,7 +1413,7 @@ BEGIN
                   FOR i IN 1..array_upper(relation.members, 1) BY 2 LOOP
                     IF NEW.parent_place_id IS NULL AND relation.members[i+1] = 'street' THEN
     --RAISE WARNING 'node in way that is in a relation %',relation;
-                      SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::integer 
+                      SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::bigint 
                         and rank_search = 26 INTO NEW.parent_place_id;
                     END IF;
                   END LOOP;
@@ -1458,7 +1458,7 @@ BEGIN
             FOR i IN 1..array_upper(relation.members, 1) BY 2 LOOP
               IF NEW.parent_place_id IS NULL AND relation.members[i+1] = 'street' THEN
 --RAISE WARNING 'way that is in a relation %',relation;
-                SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::integer
+                SELECT place_id from placex where osm_type='W' and osm_id = substring(relation.members[i],2,200)::bigint
                   and rank_search = 26 INTO NEW.parent_place_id;
               END IF;
             END LOOP;
@@ -1851,7 +1851,7 @@ BEGIN
   --DEBUG: RAISE WARNING 'placex_delete:08 % %',OLD.osm_type,OLD.osm_id;
 
   IF OLD.rank_address < 26 THEN
-    b := deleteLocationArea(OLD.partition, OLD.place_id);
+    b := deleteLocationArea(OLD.partition, OLD.place_id, OLD.rank_search);
   END IF;
 
   --DEBUG: RAISE WARNING 'placex_delete:09 % %',OLD.osm_type,OLD.osm_id;
@@ -2254,7 +2254,7 @@ DROP TYPE addressline CASCADE;
 create type addressline as (
   place_id BIGINT,
   osm_type CHAR(1),
-  osm_id INTEGER,
+  osm_id BIGINT,
   name HSTORE,
   class TEXT,
   type TEXT,

sql/partition-functions.src.sql

@@ -1,62 +1,3 @@
-drop type nearplace cascade;
-create type nearplace as (
-  place_id BIGINT
-);
-
-drop type nearfeature cascade;
-create type nearfeature as (
-  place_id BIGINT,
-  keywords int[],
-  rank_address integer,
-  rank_search integer,
-  distance float,
-  isguess boolean
-);
-
-drop type nearfeaturecentr cascade;
-create type nearfeaturecentr as (
-  place_id BIGINT,
-  keywords int[],
-  rank_address integer,
-  rank_search integer,
-  distance float,
-  isguess boolean,
-  centroid GEOMETRY
-);
-
-CREATE TABLE location_area_country () INHERITS (location_area_large);
-CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry);
-
-CREATE TABLE search_name_country () INHERITS (search_name_blank);
-CREATE INDEX idx_search_name_country_place_id ON search_name_country USING BTREE (place_id);
-CREATE INDEX idx_search_name_country_name_vector ON search_name_country USING GIN (name_vector) WITH (fastupdate = off);
-
--- start
-CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large);
-CREATE INDEX idx_location_area_large_-partition-_place_id ON location_area_large_-partition- USING BTREE (place_id);
-CREATE INDEX idx_location_area_large_-partition-_geometry ON location_area_large_-partition- USING GIST (geometry);
-
-CREATE TABLE search_name_-partition- () INHERITS (search_name_blank);
-CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id);
-CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid);
-CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off);
-
-CREATE TABLE location_property_-partition- () INHERITS (location_property);
-CREATE INDEX idx_location_property_-partition-_place_id ON location_property_-partition- USING BTREE (place_id);
-CREATE INDEX idx_location_property_-partition-_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id);
-CREATE INDEX idx_location_property_-partition-_housenumber_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id, housenumber);
-
-CREATE TABLE location_road_-partition- (
-  partition integer,
-  place_id BIGINT,
-  country_code VARCHAR(2)
-  );
-SELECT AddGeometryColumn('location_road_-partition-', 'geometry', 4326, 'GEOMETRY', 2);
-CREATE INDEX idx_location_road_-partition-_geometry ON location_road_-partition- USING GIST (geometry);
-CREATE INDEX idx_location_road_-partition-_place_id ON location_road_-partition- USING BTREE (place_id);
-
--- end
-
 create or replace function getNearFeatures(in_partition INTEGER, point GEOMETRY, maxrank INTEGER, isin_tokens INT[]) RETURNS setof nearfeaturecentr AS $$
 DECLARE
   r nearfeaturecentr%rowtype;
@@ -90,10 +31,15 @@ END
 $$
 LANGUAGE plpgsql;
 
-create or replace function deleteLocationArea(in_partition INTEGER, in_place_id BIGINT) RETURNS BOOLEAN AS $$
+create or replace function deleteLocationArea(in_partition INTEGER, in_place_id BIGINT, in_rank_search INTEGER) RETURNS BOOLEAN AS $$
 DECLARE
 BEGIN
 
+  IF in_rank_search <= 4 THEN
+    DELETE from location_area_country WHERE place_id = in_place_id;
+    RETURN TRUE;
+  END IF;
+
 -- start
   IF in_partition = -partition- THEN
     DELETE from location_area_large_-partition- WHERE place_id = in_place_id;

sql/partition-tables.src.sql

@@ -0,0 +1,58 @@
+drop type nearplace cascade;
+create type nearplace as (
+  place_id BIGINT
+);
+
+drop type nearfeature cascade;
+create type nearfeature as (
+  place_id BIGINT,
+  keywords int[],
+  rank_address integer,
+  rank_search integer,
+  distance float,
+  isguess boolean
+);
+
+drop type nearfeaturecentr cascade;
+create type nearfeaturecentr as (
+  place_id BIGINT,
+  keywords int[],
+  rank_address integer,
+  rank_search integer,
+  distance float,
+  isguess boolean,
+  centroid GEOMETRY
+);
+
+CREATE TABLE location_area_country () INHERITS (location_area_large);
+CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry);
+
+CREATE TABLE search_name_country () INHERITS (search_name_blank);
+CREATE INDEX idx_search_name_country_place_id ON search_name_country USING BTREE (place_id);
+CREATE INDEX idx_search_name_country_name_vector ON search_name_country USING GIN (name_vector) WITH (fastupdate = off);
+
+-- start
+CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large);
+CREATE INDEX idx_location_area_large_-partition-_place_id ON location_area_large_-partition- USING BTREE (place_id);
+CREATE INDEX idx_location_area_large_-partition-_geometry ON location_area_large_-partition- USING GIST (geometry);
+
+CREATE TABLE search_name_-partition- () INHERITS (search_name_blank);
+CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id);
+CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid);
+CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off);
+
+CREATE TABLE location_property_-partition- () INHERITS (location_property);
+CREATE INDEX idx_location_property_-partition-_place_id ON location_property_-partition- USING BTREE (place_id);
+CREATE INDEX idx_location_property_-partition-_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id);
+CREATE INDEX idx_location_property_-partition-_housenumber_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id, housenumber);
+
+CREATE TABLE location_road_-partition- (
+  partition integer,
+  place_id BIGINT,
+  country_code VARCHAR(2)
+  );
+SELECT AddGeometryColumn('location_road_-partition-', 'geometry', 4326, 'GEOMETRY', 2);
+CREATE INDEX idx_location_road_-partition-_geometry ON location_road_-partition- USING GIST (geometry);
+CREATE INDEX idx_location_road_-partition-_place_id ON location_road_-partition- USING BTREE (place_id);
+
+-- end

utils/setup.php

@@ -24,13 +24,14 @@
 		array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
 		array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
 		array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
-		array('create-partitions', '', 0, 1, 0, 0, 'bool', 'Create required partition tables and triggers'),
+		array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
+		array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
 		array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
 		array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
 		array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
 		array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
 		array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
-		array('create-roads', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
+		array('create-roads', '', 0, 1, 0, 0, 'bool', ''),
 		array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
 		array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
 		array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
@@ -61,7 +62,6 @@
 		}
 	}
 
-
 	// This is a pretty hard core default - the number of processors in the box - 1
 	$iInstances = isset($aCMDResult['threads'])?$aCMDResult['threads']:(getProcessorCount()-1);
 	if ($iInstances < 1)
@@ -105,6 +105,15 @@
 		// TODO: path detection, detection memory, etc.
 
 		$oDB =& getDB();
+
+                $sVersionString = $oDB->getOne('select version()');
+                preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[.]([0-9]+) #', $sVersionString, $aMatches);
+                if (((float)($aMatches[1].'.'.$aMatches[2])) > 9.1)
+                {
+                        echo "ERROR: PostgreSQL versions > 9.1 not supported.  Please use the latest development version from github.\n";
+                        exit;
+                }
+
 		passthru('createlang plpgsql '.$aDSNInfo['database']);
 		$pgver = (float) CONST_Postgresql_Version;
 		if ($pgver < 9.1) {
@@ -114,6 +123,15 @@
 		}
 		pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
 		pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
+
+                $sVersionString = $oDB->getOne('select postgis_full_version()');
+                preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
+                if (((float)($aMatches[1].'.'.$aMatches[2])) >= 2.0)
+                {
+                        echo "ERROR: PostGIS version >= 2 is not supported.  Please use the latest development version from github.\n";
+                        exit;
+                }
+
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
 		pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql');
@@ -202,9 +220,9 @@
 		pgsqlRunScript($sTemplate);
 	}
 
-	if ($aCMDResult['create-partitions'] || $aCMDResult['all'])
+	if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
 	{
-		echo "Partitions\n";
+		echo "Partition Tables\n";
 		$bDidSomething = true;
 		$oDB =& getDB();
 		$sSQL = 'select partition from country_name order by country_code';
@@ -215,7 +233,36 @@
 		}
 		$aPartitions[] = 0;
 
-		$sTemplate = file_get_contents(CONST_BasePath.'/sql/partitions.src.sql');
+		$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
+		preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
+		foreach($aMatches as $aMatch)
+		{
+			$sResult = '';
+			foreach($aPartitions as $sPartitionName)
+			{
+				$sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
+			}
+			$sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
+		}
+
+		pgsqlRunScript($sTemplate);
+	}
+
+
+	if ($aCMDResult['create-partition-functions'] || $aCMDResult['all'])
+	{
+		echo "Partition Functions\n";
+		$bDidSomething = true;
+		$oDB =& getDB();
+		$sSQL = 'select partition from country_name order by country_code';
+		$aPartitions = $oDB->getCol($sSQL);
+		if (PEAR::isError($aPartitions))
+		{
+			fail($aPartitions->getMessage());
+		}
+		$aPartitions[] = 0;
+
+		$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
 		preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
 		foreach($aMatches as $aMatch)
 		{