Merge pull request #1433 from mtmail/us-postcode-import-optional

make US postcode data to an optional download

data/us_postcode_table.sql

@@ -0,0 +1,16 @@
+SET statement_timeout = 0;
+SET client_encoding = 'UTF8';
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+
+SET search_path = public, pg_catalog;
+
+SET default_tablespace = '';
+
+SET default_with_oids = false;
+
+CREATE TABLE us_postcode (
+    postcode text,
+    x double precision,
+    y double precision
+);

docs/admin/Import-and-Update.md

@@ -51,12 +51,14 @@ size of Nominatim. They also increase the install time by an hour or so.
 the initial import of the data if you want the rankings applied to the
 loaded data.
 
-### UK postcodes
+### Great Britain, USA postcodes
 
-Nominatim can use postcodes from an external source to improve searches that involve a UK postcode. This data can be optionally downloaded:
+Nominatim can use postcodes from an external source to improve searches that
+involve a GB or US postcode. This data can be optionally downloaded:
 
     cd $NOMINATIM_SOURCE_DIR/data
     wget https://www.nominatim.org/data/gb_postcode_data.sql.gz
+    wget https://www.nominatim.org/data/us_postcode_data.sql.gz
 
 ## Choosing the Data to Import
 

lib/setup/SetupClass.php

@@ -139,16 +139,20 @@ class SetupFunctions
         $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
         $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz');
         $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
+        $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode_table.sql');
 
         $sPostcodeFilename = CONST_BasePath.'/data/gb_postcode_data.sql.gz';
         if (file_exists($sPostcodeFilename)) {
             $this->pgsqlRunScriptFile($sPostcodeFilename);
         } else {
-            warn('optional external UK postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
+            warn('optional external GB postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
         }
 
-        if (CONST_Use_Extra_US_Postcodes) {
+        $sPostcodeFilename = CONST_BasePath.'/data/us_postcode_data.sql.gz';
-            $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
+        if (file_exists($sPostcodeFilename)) {
+            $this->pgsqlRunScriptFile($sPostcodeFilename);
+        } else {
+            warn('optional external US postcode table file ('.$sPostcodeFilename.') not found. Skipping.');
         }
 
         if ($this->bNoPartitions) {
@@ -558,17 +562,15 @@ class SetupFunctions
         $sSQL .= ' GROUP BY country_code, pc';
         $this->pgExec($sSQL);
 
-        if (CONST_Use_Extra_US_Postcodes) {
+        // only add postcodes that are not yet available in OSM
-            // only add postcodes that are not yet available in OSM
+        $sSQL  = 'INSERT INTO location_postcode';
-            $sSQL  = 'INSERT INTO location_postcode';
+        $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
-            $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
+        $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
-            $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
+        $sSQL .= '       ST_SetSRID(ST_Point(x,y),4326)';
-            $sSQL .= '       ST_SetSRID(ST_Point(x,y),4326)';
+        $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
-            $sSQL .= '  FROM us_postcode WHERE postcode NOT IN';
+        $sSQL .= '        (SELECT postcode FROM location_postcode';
-            $sSQL .= '        (SELECT postcode FROM location_postcode';
+        $sSQL .= "          WHERE country_code = 'us')";
-            $sSQL .= "          WHERE country_code = 'us')";
+        $this->pgExec($sSQL);
-            $this->pgExec($sSQL);
-        }
 
         // add missing postcodes for GB (if available)
         $sSQL  = 'INSERT INTO location_postcode';

settings/defaults.php

@@ -23,8 +23,6 @@ if (isset($_GET['debug']) && $_GET['debug']) @define('CONST_Debug', true);
 // term. Spaces are kept but collapsed to one standard space.
 @define('CONST_Term_Normalization_Rules', ":: NFD (); [[:Nonspacing Mark:] [:Cf:]] >;  :: lower (); [[:Punctuation:][:Space:]]+ > ' '; :: NFC ();");
 
-// Set to false to avoid importing extra postcodes for the US.
-@define('CONST_Use_Extra_US_Postcodes', true);
 /* Set to true after importing Tiger house number data for the US.
    Note: The tables must already exist or queries will throw errors.
          After changing this setting run ./utils/setup --create-functions

sql/tables.sql

@@ -36,6 +36,7 @@ GRANT SELECT ON new_query_log TO "{www-user}" ;
 
 GRANT SELECT ON TABLE country_name TO "{www-user}";
 GRANT SELECT ON TABLE gb_postcode TO "{www-user}";
+GRANT SELECT ON TABLE us_postcode TO "{www-user}";
 
 drop table IF EXISTS word;
 CREATE TABLE word (