From de45bafc5455c3ccf4c8fd082abef9ee8dfe9d3d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 17 Sep 2013 21:26:07 +0200 Subject: [PATCH 01/29] force delete from place if there is no corresponding object in placex This might happen for nameless landuse/natural objects that are added to place during initial import but then dropped when being copied to placex. If they later receive a name, thus becoming valid, then place_insert should delete the orphan object in place and reinsert it. If they are large enough, the place_delete trigger prevents them from being removed. The additional update fools the delete trigger. --- sql/functions.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/functions.sql b/sql/functions.sql index 6d2ae5f8..48720628 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -2097,6 +2097,9 @@ BEGIN IF existingplacex.osm_type IS NULL THEN IF existing.osm_type IS NOT NULL THEN + -- pathological case caused by the triggerless copy into place during initial import + -- force delete even for large areas, it will be reinserted later + UPDATE place set geometry = ST_SetSRID(ST_Point(0,0), 4326) where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type; DELETE from place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class and type = NEW.type; END IF; From f723eb199836887a535942c387091557a76f88aa Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 19 Sep 2013 22:08:22 +0200 Subject: [PATCH 02/29] move blocking of highway objects into osm2pgsql --- osm2pgsql | 2 +- sql/functions.sql | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/osm2pgsql b/osm2pgsql index 0005c331..48377cec 160000 --- a/osm2pgsql +++ b/osm2pgsql @@ -1 +1 @@ -Subproject commit 0005c3312cdbddb51fdb749315d4c7df59daa82a +Subproject commit 48377cecdccacc64528fe070a1495cb2070e833a diff --git a/sql/functions.sql b/sql/functions.sql index 48720628..09ff155c 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -938,10 +938,6 @@ BEGIN --DEBUG: RAISE WARNING '% %',NEW.osm_type,NEW.osm_id; -- just block these - IF NEW.class = 'highway' and NEW.type in ('turning_circle','traffic_signals','mini_roundabout','noexit','crossing') THEN --- RAISE WARNING 'bad highway %',NEW.osm_id; - RETURN null; - END IF; IF NEW.class in ('landuse','natural') and NEW.name is null THEN -- RAISE WARNING 'empty landuse %',NEW.osm_id; RETURN null; @@ -2044,9 +2040,6 @@ BEGIN END IF; -- Just block these - lots and pointless - IF NEW.class = 'highway' and NEW.type in ('turning_circle','traffic_signals','mini_roundabout','noexit','crossing') THEN - RETURN null; - END IF; IF NEW.class in ('landuse','natural') and NEW.name is null THEN RETURN null; END IF; From 8332fedfd6790ed74fba981472a8885e1711fa2a Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 19 Sep 2013 22:56:55 +0200 Subject: [PATCH 03/29] make sleep interval dependent on date of last batch --- utils/update.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/update.php b/utils/update.php index d8509b37..ba0a5c26 100755 --- a/utils/update.php +++ b/utils/update.php @@ -531,7 +531,7 @@ if ( CONST_Replication_Update_Interval > 60 ) { - $iSleep = round(CONST_Replication_Update_Interval*0.8); + $iSleep = max(0,(strtotime($sBatchEnd)+CONST_Replication_Update_Interval-time())); } else { From e3aa2ea78772525bb875edf76169e9bba43942ef Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 19 Sep 2013 23:42:15 +0200 Subject: [PATCH 04/29] restrict max admin_level when patching in tag changes into placex --- sql/functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/functions.sql b/sql/functions.sql index 09ff155c..5bbaad5b 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -2251,7 +2251,7 @@ BEGIN country_code = NEW.country_code, parent_place_id = null, extratags = NEW.extratags, - admin_level = NEW.admin_level, + admin_level = CASE WHEN NEW.admin_level > 15 THEN 15 ELSE NEW.admin_level END, indexed_status = 2, geometry = NEW.geometry where place_id = existingplacex.place_id; From ef834a9139e3b93b9d6ed2207bb81f36bb35a61c Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 23 Sep 2013 20:50:13 +0200 Subject: [PATCH 05/29] update authors and change log for pending release --- AUTHORS | 15 ++++++++++++++- ChangeLog | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 704d6bed..1769f616 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,14 @@ -Nominatim was written by Brian Quinion. +Nominatim was written by: + + Brian Quinion + Sarah Hoffmann + Frederik Ramm + Michael Spreng + Daniele Forsi + mfn + Grant Slater + Andree Klattenhoff + IrlJidel + appelflap + b3nn0 + Spin0us diff --git a/ChangeLog b/ChangeLog index e69de29b..19a6a760 100644 --- a/ChangeLog +++ b/ChangeLog @@ -0,0 +1,28 @@ +2.0.1 + + * delete outdated entries from location_area_country + * remove remaining uses of INTEGER, to allow node ids larger than 2^31 + +2.1 + + * update to openlayers 2.12 (now custom built) + * update fallback OSM boundaries + * add support for postgresql 9.2/9.3 and postgis 2.x + * add structured queries + * add options for polygon output in various formats + (geojson, svg, kml, postgis text) + * maintenance functions for deleting objects and updating regions + (plcae_force_update/place_force_delete) + * web view for polygons that need deleting + * rate limiting using memcache + * improve layout of details page + * add support for boundary:postal_code + * full CORS support + * improve parenting of POIs + * support for extract daily diffs from Geofabrik + * support for addresses without a street + (addr:place and conscription number house numbers) + * improve layout of word and search_name_* tables + * support for US ZIP+4 codes + * refactoring of front-end PHP code + * lots of smaller bug fixes From 02f02c4ca4c0b30e83033a4179cb6f94192ab743 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 23 Sep 2013 20:51:44 +0200 Subject: [PATCH 06/29] update osm2pgsql (failing updates) --- osm2pgsql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osm2pgsql b/osm2pgsql index 48377cec..86d1e0ed 160000 --- a/osm2pgsql +++ b/osm2pgsql @@ -1 +1 @@ -Subproject commit 48377cecdccacc64528fe070a1495cb2070e833a +Subproject commit 86d1e0ede1e01593a3d1b4f2284e9c0bab873ce4 From 528de055d17099746d7d1220cb508c54d69dd0fd Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 25 Sep 2013 22:55:47 +0200 Subject: [PATCH 07/29] make html validator happy --- lib/template/details-html.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/template/details-html.php b/lib/template/details-html.php index ba41b931..83df4814 100644 --- a/lib/template/details-html.php +++ b/lib/template/details-html.php @@ -80,7 +80,7 @@ echo '

'; if ($aPointDetails['icon']) { - echo ''; + echo ''.$aPointDetails['icon'].''; } echo $aPointDetails['localname'].'

'; echo '
'; From bf182ba2dc5fda3b249bf192a8c488e564576fd7 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 25 Sep 2013 23:05:47 +0200 Subject: [PATCH 08/29] make import-osmosis(-all) fail with error code!=0 --- utils/update.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/update.php b/utils/update.php index ba0a5c26..533b48d6 100755 --- a/utils/update.php +++ b/utils/update.php @@ -446,7 +446,7 @@ if ($iErrorLevel) { echo "Error: $iErrorLevel\n"; - exit; + exit($iErrorLevel); } echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')"; @@ -469,7 +469,7 @@ if (PEAR::isError($iFileID)) { echo $iFileID->getMessage()."\n"; - exit; + exit(-1); } $sFileDir = CONST_BasePath.'/export/diff/'; $sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT); @@ -495,7 +495,7 @@ if ($iErrorLevel) { echo "Error: $iErrorLevel\n"; - exit; + exit($iErrorLevel); } if (!$aResult['no-npi']) @@ -510,7 +510,7 @@ if ($iErrorLevel) { echo "Error: $iErrorLevel\n"; - exit; + exit($iErrorLevel); } rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2", From 2df74352f8082968f94700eaf8eee9b4a62e33ed Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 26 Sep 2013 21:19:29 +0200 Subject: [PATCH 09/29] new development release 2.2 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index e89b2217..a92bbeeb 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(Nominatim,2.1) +AC_INIT(Nominatim,2.2) if git rev-parse HEAD 2>/dev/null >/dev/null; then AC_SUBST([PACKAGE_VERSION], [$PACKAGE_VERSION-git-`git rev-parse --short HEAD`]) fi From 1c7afef1f2195ab5548dd92bd013217a33691562 Mon Sep 17 00:00:00 2001 From: Philippe Date: Fri, 27 Sep 2013 17:59:42 -0400 Subject: [PATCH 10/29] added FIPS codes for 60xxx, 66xxx, 69xxx, 72xxx and 78xxx --- utils/tigerAddressImport.py | 96 +++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/utils/tigerAddressImport.py b/utils/tigerAddressImport.py index a784b93b..c4bca715 100755 --- a/utils/tigerAddressImport.py +++ b/utils/tigerAddressImport.py @@ -3325,6 +3325,102 @@ county_fips = { '56041' : 'Uinta, WY' , '56043' : 'Washakie, WY' , '56045' : 'Weston, WY' , +'60000' : 'AMERICAN SAMOA', +'60010' : 'Eastern district, AS' , +'60020' : 'Manua district, AS' , +'60030' : 'Rose island, AS' , +'60040' : 'Swains island, AS' , +'60050' : 'Western district, AS' , +'66000' : 'GUAM' , +'66010' : 'Guam, GU' , +'69000' : 'COMMONWEALTH OF THE NORTHERN MARIANA ISLANDS' , +'69085' : 'Commonwealth of the Northern Mariana Islands, MP' , +'69100' : 'Commonwealth of the Northern Mariana Islands, MP' , +'69110' : 'Commonwealth of the Northern Mariana Islands, MP' , +'69120' : 'Commonwealth of the Northern Mariana Islands, MP' , +'72000' : 'PUERTO RICO', +'72001' : 'Adjuntas, PR', +'72003' : 'Aguada, PR', +'72005' : 'Aguadilla, PR', +'72007' : 'Aguas Buenas, PR', +'72009' : 'Aibonito, PR', +'72011' : 'Anasco, PR', +'72013' : 'Arecibo, PR', +'72015' : 'Arroyo, PR', +'72017' : 'Barceloneta, PR', +'72019' : 'Barranquitas, PR', +'72021' : 'Bayamon, PR', +'72023' : 'Cabo Rojo, PR', +'72025' : 'Caguas, PR', +'72027' : 'Camuy, PR', +'72029' : 'Canovanas, PR', +'72031' : 'Carolina, PR', +'72033' : 'Catano, PR', +'72035' : 'Cayey, PR', +'72037' : 'Ceiba, PR', +'72039' : 'Ciales, PR', +'72041' : 'Cidra, PR', +'72043' : 'Coamo, PR', +'72045' : 'Comerio, PR', +'72047' : 'Corozal, PR', +'72049' : 'Culebra, PR', +'72051' : 'Dorado, PR', +'72053' : 'Fajardo, PR', +'72054' : 'Florida, PR', +'72055' : 'Guanica, PR', +'72057' : 'Guayama, PR', +'72059' : 'Guayanilla, PR', +'72061' : 'Guaynabo, PR', +'72063' : 'Gurabo, PR', +'72065' : 'Hatillo, PR', +'72067' : 'Hormigueros, PR', +'72069' : 'Humacao, PR', +'72071' : 'Isabela, PR', +'72073' : 'Jayuya, PR', +'72075' : 'Juana Diaz, PR', +'72077' : 'Juncos, PR', +'72079' : 'Lajas, PR', +'72081' : 'Lares, PR', +'72083' : 'Las Marias, PR', +'72085' : 'Las Piedras, PR', +'72087' : 'Loiza, PR', +'72089' : 'Luquillo, PR', +'72091' : 'Manati, PR', +'72093' : 'Maricao, PR', +'72095' : 'Maunabo, PR', +'72097' : 'Mayaguez, PR', +'72099' : 'Moca, PR', +'72101' : 'Morovis, PR', +'72103' : 'Naguabo, PR', +'72105' : 'Naranjito, PR', +'72107' : 'Orocovis, PR', +'72109' : 'Patillas, PR', +'72111' : 'Penuelas, PR', +'72113' : 'Ponce, PR', +'72115' : 'Quebradillas, PR', +'72117' : 'Rincon, PR', +'72119' : 'Rio Grande, PR', +'72121' : 'Sabana Grande, PR', +'72123' : 'Salinas, PR', +'72125' : 'San German, PR', +'72127' : 'San Juan, PR', +'72129' : 'San Lorenzo, PR', +'72131' : 'San Sebastian, PR', +'72133' : 'Santa Isabel, PR', +'72135' : 'Toa Alta, PR', +'72137' : 'Toa Baja, PR', +'72139' : 'Trujillo Alto, PR', +'72141' : 'Utuado, PR', +'72143' : 'Vega Alta, PR', +'72145' : 'Vega Baja, PR', +'72147' : 'Vieques, PR', +'72149' : 'Villalba, PR', +'72151' : 'Yabucoa, PR', +'72153' : 'Yauco, PR', +'78000' : 'VIRGIN ISLANDS' , +'78010' : 'St. Croix, VI' , +'78020' : 'St. John, VI' , +'78030' : 'St. Thomas, VI' , } def fipsstate(fips,countyfp): From b80c1cc774339b99d4329a4002f9d9ccfd033038 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 29 Sep 2013 17:22:21 +0200 Subject: [PATCH 11/29] revoke most of www-data's write rights --- sql/tables.sql | 7 ------- 1 file changed, 7 deletions(-) diff --git a/sql/tables.sql b/sql/tables.sql index 3bb41e28..038a373e 100644 --- a/sql/tables.sql +++ b/sql/tables.sql @@ -212,16 +212,9 @@ CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name DROP SEQUENCE seq_place; CREATE SEQUENCE seq_place start 1; GRANT SELECT on placex to "www-data" ; -GRANT UPDATE ON placex to "www-data" ; GRANT SELECT ON search_name to "www-data" ; -GRANT DELETE on search_name to "www-data" ; -GRANT INSERT on search_name to "www-data" ; GRANT SELECT on place_addressline to "www-data" ; -GRANT INSERT ON place_addressline to "www-data" ; -GRANT DELETE on place_addressline to "www-data" ; GRANT SELECT ON seq_word to "www-data" ; -GRANT UPDATE ON seq_word to "www-data" ; -GRANT INSERT ON word to "www-data" ; GRANT SELECT ON planet_osm_ways to "www-data" ; GRANT SELECT ON planet_osm_rels to "www-data" ; GRANT SELECT on location_area to "www-data" ; From 7471b09ed376f0870fad08234e89990e5cb74e02 Mon Sep 17 00:00:00 2001 From: IrlJidel Date: Mon, 30 Sep 2013 13:20:24 +0100 Subject: [PATCH 12/29] set start time for indexing step --- utils/update.php | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/update.php b/utils/update.php index 533b48d6..ac7b5be8 100755 --- a/utils/update.php +++ b/utils/update.php @@ -461,6 +461,7 @@ // Index file $sThisIndexCmd = $sCMDIndex; + $fCMDStartTime = time(); if (!$aResult['no-npi']) { From 84d1e521aac266175c039712aa5a03e5701a3cfa Mon Sep 17 00:00:00 2001 From: IrlJidel Date: Mon, 30 Sep 2013 13:25:30 +0100 Subject: [PATCH 13/29] show import_osmosis_log insert statements, print completed log after import_status inserts --- utils/update.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/utils/update.php b/utils/update.php index ac7b5be8..069d9e72 100755 --- a/utils/update.php +++ b/utils/update.php @@ -431,9 +431,10 @@ } $iFileSize = filesize($sImportFile); $sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory); - echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')"; + var_Dump($sSQL); $oDB->query($sSQL); + echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; } $iFileSize = filesize($sImportFile); @@ -452,6 +453,7 @@ $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')"; var_Dump($sSQL); $oDB->query($sSQL); + echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; // Archive for debug? unlink($sImportFile); @@ -461,7 +463,7 @@ // Index file $sThisIndexCmd = $sCMDIndex; - $fCMDStartTime = time(); + $fCMDStartTime = time(); if (!$aResult['no-npi']) { @@ -519,9 +521,10 @@ } } - echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')"; + var_Dump($sSQL); $oDB->query($sSQL); + echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "update import_status set lastimportdate = '$sBatchEnd'"; $oDB->query($sSQL); From 1c86275bade6d39d0c44569fe88bb63def5531b8 Mon Sep 17 00:00:00 2001 From: IrlJidel Date: Mon, 30 Sep 2013 13:44:50 +0100 Subject: [PATCH 14/29] output date of steps during import-osmosis --- utils/update.php | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/utils/update.php b/utils/update.php index 069d9e72..7fa8e73c 100755 --- a/utils/update.php +++ b/utils/update.php @@ -417,7 +417,7 @@ exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel); } // There are new replication files - use osmosis to download the file - echo "\nReplication Delay is ".$aReplicationLag[0]."\n"; + echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n"; } $fCMDStartTime = time(); echo $sCMDDownload."\n"; @@ -434,7 +434,7 @@ $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')"; var_Dump($sSQL); $oDB->query($sSQL); - echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; + echo date('Y-m-d H:i:s')." Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; } $iFileSize = filesize($sImportFile); @@ -449,11 +449,11 @@ echo "Error: $iErrorLevel\n"; exit($iErrorLevel); } - echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; + echo date('Y-m-d H:i:s')." Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')"; var_Dump($sSQL); $oDB->query($sSQL); - echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; + echo date('Y-m-d H:i:s')." Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; // Archive for debug? unlink($sImportFile); @@ -467,7 +467,6 @@ if (!$aResult['no-npi']) { - $fCMDStartTime = time(); $iFileID = $oDB->getOne('select nextval(\'file\')'); if (PEAR::isError($iFileID)) { @@ -524,13 +523,13 @@ $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')"; var_Dump($sSQL); $oDB->query($sSQL); - echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; + echo date('Y-m-d H:i:s')." Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "update import_status set lastimportdate = '$sBatchEnd'"; $oDB->query($sSQL); $fDuration = time() - $fStartTime; - echo "Completed for $sBatchEnd in ".round($fDuration/60,2)."\n"; + echo date('Y-m-d H:i:s')." Completed for $sBatchEnd in ".round($fDuration/60,2)."\n"; if (!$aResult['import-osmosis-all']) exit; if ( CONST_Replication_Update_Interval > 60 ) @@ -541,7 +540,7 @@ { $iSleep = max(0,CONST_Replication_Update_Interval-$fDuration); } - echo "Sleeping $iSleep seconds\n"; + echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n"; sleep($iSleep); } From dbd7f8fdaebcd768989c6e3ba40738fcb0cdde1b Mon Sep 17 00:00:00 2001 From: IrlJidel Date: Wed, 2 Oct 2013 10:53:21 +0100 Subject: [PATCH 15/29] dont include polling time for batch duration for non-minutely updates, log name of each update step --- utils/update.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils/update.php b/utils/update.php index 7fa8e73c..42a7b59e 100755 --- a/utils/update.php +++ b/utils/update.php @@ -419,6 +419,7 @@ // There are new replication files - use osmosis to download the file echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n"; } + $fStartTime = time(); $fCMDStartTime = time(); echo $sCMDDownload."\n"; exec($sCMDDownload, $sJunk, $iErrorLevel); @@ -434,7 +435,7 @@ $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')"; var_Dump($sSQL); $oDB->query($sSQL); - echo date('Y-m-d H:i:s')." Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; + echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; } $iFileSize = filesize($sImportFile); @@ -449,11 +450,10 @@ echo "Error: $iErrorLevel\n"; exit($iErrorLevel); } - echo date('Y-m-d H:i:s')." Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')"; var_Dump($sSQL); $oDB->query($sSQL); - echo date('Y-m-d H:i:s')." Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; + echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; // Archive for debug? unlink($sImportFile); @@ -523,13 +523,13 @@ $sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')"; var_Dump($sSQL); $oDB->query($sSQL); - echo date('Y-m-d H:i:s')." Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; + echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n"; $sSQL = "update import_status set lastimportdate = '$sBatchEnd'"; $oDB->query($sSQL); $fDuration = time() - $fStartTime; - echo date('Y-m-d H:i:s')." Completed for $sBatchEnd in ".round($fDuration/60,2)."\n"; + echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60,2)." minutes\n"; if (!$aResult['import-osmosis-all']) exit; if ( CONST_Replication_Update_Interval > 60 ) From 7854075c5c0ca2d1d2d09b1a9951d103b5430058 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 2 Oct 2013 19:03:13 +0200 Subject: [PATCH 16/29] hasn't been svn for a while --- nominatim/nominatim.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nominatim/nominatim.c b/nominatim/nominatim.c index d0eeff22..7a9e6e9e 100644 --- a/nominatim/nominatim.c +++ b/nominatim/nominatim.c @@ -115,7 +115,7 @@ int main(int argc, char *argv[]) PGconn *conn; - fprintf(stderr, "nominatim SVN version %s\n\n", VERSION); + fprintf(stderr, "nominatim version %s\n\n", VERSION); while (1) { From 8675a2607a289ad1a027422730bc65982fe5221f Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 10 Oct 2013 23:10:18 +0200 Subject: [PATCH 17/29] use postgis as an extension for versions >= 2.0 --- utils/setup.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/utils/setup.php b/utils/setup.php index 0e1a1f4d..fb16254d 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -125,7 +125,13 @@ pgsqlRunScript('CREATE EXTENSION hstore'); } - pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql'); + $fPostgisVersion = (float) CONST_Postgis_Version; + if ($fPostgisVersion < 2.0) { + pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql'); + pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql'); + } else { + pgsqlRunScript('CREATE EXTENSION postgis'); + } $sVersionString = $oDB->getOne('select postgis_full_version()'); preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches); if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2]) @@ -134,7 +140,6 @@ exit; } - pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql'); From 71429b91b40921acc9c4d059b3b8e2d5c53ef07a Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Tue, 15 Oct 2013 21:10:46 +0200 Subject: [PATCH 18/29] take non-admin boundaries out of address computation fixes #90 --- sql/functions.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/functions.sql b/sql/functions.sql index 5bbaad5b..bc366f8b 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -1123,7 +1123,11 @@ BEGIN return NULL; END IF; NEW.rank_search := NEW.admin_level * 2; - NEW.rank_address := NEW.rank_search; + IF NEW.type = 'administrative' THEN + NEW.rank_address := NEW.rank_search; + ELSE + NEW.rank_address := 0; + END IF; ELSEIF NEW.class = 'landuse' AND ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') THEN NEW.rank_search := 22; NEW.rank_address := NEW.rank_search; From 3470abad079927d5ee3b659865a940b5703f8551 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 18 Oct 2013 19:23:04 +0200 Subject: [PATCH 19/29] don't send blocked IPs to OSM's trac by default --- lib/init-website.php | 3 +-- settings/settings.php | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/init-website.php b/lib/init-website.php index bcf3ebf7..e47b73fb 100644 --- a/lib/init-website.php +++ b/lib/init-website.php @@ -55,8 +55,7 @@ if (strpos(CONST_BlockedIPs, ','.$_SERVER["REMOTE_ADDR"].',') !== false || $fBucketVal >= CONST_ConnectionBucket_BlockLimit) { echo "Your IP has been blocked. \n"; - echo "Please create a nominatim trac ticket (http://trac.openstreetmap.org/newticket?component=nominatim) to request this to be removed. \n"; - echo "Information on the Nominatim usage policy can be found here: http://wiki.openstreetmap.org/wiki/Nominatim#Usage_Policy \n"; + echo CONST_BlockMessage; exit; } diff --git a/settings/settings.php b/settings/settings.php index 0ce34ad2..27b3446b 100644 --- a/settings/settings.php +++ b/settings/settings.php @@ -55,6 +55,7 @@ @define('CONST_ClosedForIndexingExceptionIPs', ''); @define('CONST_BlockedIPs', ''); @define('CONST_BulkUserIPs', ''); + @define('CONST_BlockMessage', ''); // additional info to show for blocked IPs @define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/'); @define('CONST_Tile_Default', 'Mapnik'); From 17e2e95b530018c0f090a20dfee2246acc4f4f94 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 21 Oct 2013 18:26:45 +0200 Subject: [PATCH 20/29] properly quote quotes in type field --- lib/template/search-xml.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/template/search-xml.php b/lib/template/search-xml.php index dfcf1375..693330bb 100644 --- a/lib/template/search-xml.php +++ b/lib/template/search-xml.php @@ -80,7 +80,7 @@ echo " display_name='".htmlspecialchars($aResult['name'], ENT_QUOTES)."'"; echo " class='".htmlspecialchars($aResult['class'])."'"; - echo " type='".htmlspecialchars($aResult['type'])."'"; + echo " type='".htmlspecialchars($aResult['type'], ENT_QUOTES)."'"; echo " importance='".htmlspecialchars($aResult['importance'])."'"; if (isset($aResult['icon']) && $aResult['icon']) { From bb18cc861caea017266761fcca2429d045a254b0 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 30 Oct 2013 21:09:21 +0100 Subject: [PATCH 21/29] avoid adding NULL tokens for frequent full names --- sql/functions.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/functions.sql b/sql/functions.sql index bc366f8b..4670a71f 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -347,7 +347,7 @@ BEGIN w := getorcreate_word_id(s); - IF NOT (ARRAY[w] <@ result) THEN + IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN result := result || w; END IF; @@ -415,7 +415,7 @@ BEGIN w := getorcreate_word_id(s); - IF NOT (ARRAY[w] <@ result) THEN + IF w IS NOT NULL AND NOT (ARRAY[w] <@ result) THEN result := result || w; END IF; From 45c1e7582f87d2c5ec7cc2a06bb0b2c74939cf85 Mon Sep 17 00:00:00 2001 From: Kurt Roeckx Date: Wed, 13 Nov 2013 22:24:10 +0100 Subject: [PATCH 22/29] Connect to the correct postgresql port from the settings. --- utils/update.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/utils/update.php b/utils/update.php index 42a7b59e..8dd1dfc3 100755 --- a/utils/update.php +++ b/utils/update.php @@ -74,6 +74,7 @@ $oDB =& getDB(); $aDSNInfo = DB::parseDSN(CONST_Database_DSN); + if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; // cache memory to be used by osm2pgsql, should not be more than the available memory $iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000); @@ -82,7 +83,7 @@ $iCacheMemory = getCacheMemoryMB(); echo "WARNING: resetting cache memory to $iCacheMemory\n"; } - $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database']; + $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; if (!is_null(CONST_Osm2pgsql_Flatnode_File)) { $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; @@ -360,7 +361,7 @@ if ($aResult['index']) { - passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']); + passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']); } if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) @@ -377,7 +378,7 @@ $sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile; $sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory; $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile; - $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$aResult['index-instances']; + $sCMDIndex = $sBasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances']; if (!$aResult['no-npi']) { $sCMDIndex .= '-F '; } From e9222ba4c6a522f82bfc68ed1415f533aace5aa3 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 23 Nov 2013 12:43:28 +0100 Subject: [PATCH 23/29] explain results for Geocode::lookup() --- lib/Geocode.php | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/Geocode.php b/lib/Geocode.php index 16c40f52..91f828fa 100644 --- a/lib/Geocode.php +++ b/lib/Geocode.php @@ -309,6 +309,36 @@ return $aSearchResults; } + /* Perform the actual query lookup. + + Returns an ordered list of results, each with the following fields: + osm_type: type of corresponding OSM object + N - node + W - way + R - relation + P - postcode (internally computed) + osm_id: id of corresponding OSM object + class: general object class (corresponds to tag key of primary OSM tag) + type: subclass of object (corresponds to tag value of primary OSM tag) + admin_level: see http://wiki.openstreetmap.org/wiki/Admin_level + rank_search: rank in search hierarchy + (see also http://wiki.openstreetmap.org/wiki/Nominatim/Development_overview#Country_to_street_level) + rank_address: rank in address hierarchy (determines orer in address) + place_id: internal key (may differ between different instances) + country_code: ISO country code + langaddress: localized full address + placename: localized name of object + ref: content of ref tag (if available) + lon: longitude + lat: latitude + importance: importance of place based on Wikipedia link count + addressimportance: cumulated importance of address elements + extra_place: type of place (for admin boundaries, if there is a place tag) + aBoundingBox: bounding Box + label: short description of the object class/type (English only) + name: full name (currently the same as langaddress) + foundorder: further ordering value for places with same importance + */ function lookup() { if (!$this->sQuery && !$this->aStructuredQuery) return false; From b544a8c590a9a9b5e86ac95260831bfadd7ba1d7 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 23 Nov 2013 13:14:42 +0100 Subject: [PATCH 24/29] fix indentation --- lib/Geocode.php | 58 ++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/lib/Geocode.php b/lib/Geocode.php index 91f828fa..bcc69c75 100644 --- a/lib/Geocode.php +++ b/lib/Geocode.php @@ -309,36 +309,36 @@ return $aSearchResults; } - /* Perform the actual query lookup. + /* Perform the actual query lookup. - Returns an ordered list of results, each with the following fields: - osm_type: type of corresponding OSM object - N - node - W - way - R - relation - P - postcode (internally computed) - osm_id: id of corresponding OSM object - class: general object class (corresponds to tag key of primary OSM tag) - type: subclass of object (corresponds to tag value of primary OSM tag) - admin_level: see http://wiki.openstreetmap.org/wiki/Admin_level - rank_search: rank in search hierarchy - (see also http://wiki.openstreetmap.org/wiki/Nominatim/Development_overview#Country_to_street_level) - rank_address: rank in address hierarchy (determines orer in address) - place_id: internal key (may differ between different instances) - country_code: ISO country code - langaddress: localized full address - placename: localized name of object - ref: content of ref tag (if available) - lon: longitude - lat: latitude - importance: importance of place based on Wikipedia link count - addressimportance: cumulated importance of address elements - extra_place: type of place (for admin boundaries, if there is a place tag) - aBoundingBox: bounding Box - label: short description of the object class/type (English only) - name: full name (currently the same as langaddress) - foundorder: further ordering value for places with same importance - */ + Returns an ordered list of results, each with the following fields: + osm_type: type of corresponding OSM object + N - node + W - way + R - relation + P - postcode (internally computed) + osm_id: id of corresponding OSM object + class: general object class (corresponds to tag key of primary OSM tag) + type: subclass of object (corresponds to tag value of primary OSM tag) + admin_level: see http://wiki.openstreetmap.org/wiki/Admin_level + rank_search: rank in search hierarchy + (see also http://wiki.openstreetmap.org/wiki/Nominatim/Development_overview#Country_to_street_level) + rank_address: rank in address hierarchy (determines orer in address) + place_id: internal key (may differ between different instances) + country_code: ISO country code + langaddress: localized full address + placename: localized name of object + ref: content of ref tag (if available) + lon: longitude + lat: latitude + importance: importance of place based on Wikipedia link count + addressimportance: cumulated importance of address elements + extra_place: type of place (for admin boundaries, if there is a place tag) + aBoundingBox: bounding Box + label: short description of the object class/type (English only) + name: full name (currently the same as langaddress) + foundorder: further ordering value for places with same importance + */ function lookup() { if (!$this->sQuery && !$this->aStructuredQuery) return false; From cbbcc2618fe6bb79a8d7ddfb84c806d90631ff33 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 23 Nov 2013 13:31:29 +0100 Subject: [PATCH 25/29] make classtype table update work when special schemas are used patch from https://trac.openstreetmap.org/ticket/5037 --- sql/functions.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/functions.sql b/sql/functions.sql index 4670a71f..73aa4fee 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -1245,7 +1245,7 @@ BEGIN -- Note: won't work on initial import because the classtype tables -- do not yet exist. It won't hurt either. classtable := 'place_classtype_' || NEW.class || '_' || NEW.type; - SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable INTO result; + SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable and schemaname = current_schema() INTO result; IF result THEN EXECUTE 'INSERT INTO ' || classtable::regclass || ' (place_id, centroid) VALUES ($1,$2)' USING NEW.place_id, ST_Centroid(NEW.geometry); @@ -1987,7 +1987,7 @@ BEGIN -- remove from tables for special search classtable := 'place_classtype_' || OLD.class || '_' || OLD.type; - SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable INTO b; + SELECT count(*)>0 FROM pg_tables WHERE tablename = classtable and schemaname = current_schema() INTO b; IF b THEN EXECUTE 'DELETE FROM ' || classtable::regclass || ' WHERE place_id = $1' USING OLD.place_id; END IF; From 08eff3c482ce1c7479b4b1b5548276a56f9cffba Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 4 Dec 2013 07:53:53 +0100 Subject: [PATCH 26/29] further tweak secondary ordering of results - make address importance work for house number onjects - add number of exact matches - add importance of class/type --- lib/Geocode.php | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/lib/Geocode.php b/lib/Geocode.php index bcc69c75..48518dd3 100644 --- a/lib/Geocode.php +++ b/lib/Geocode.php @@ -32,6 +32,7 @@ protected $iMinAddressRank = 0; protected $iMaxAddressRank = 30; protected $aAddressRankList = array(); + protected $exactMatchCache = array(); protected $sAllowedTypesSQLList = false; @@ -244,13 +245,13 @@ // Get the details for display (is this a redundant extra step?) $sPlaceIDs = join(',',$aPlaceIDs); - $sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id,calculated_country_code as country_code,"; + $sSQL = "select osm_type,osm_id,class,type,admin_level,rank_search,rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id, calculated_country_code as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; $sSQL .= "get_name_by_language(name, ARRAY['ref']) as ref,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= "coalesce(importance,0.75-(rank_search::float/40)) as importance, "; - $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(placex.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; + $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(CASE WHEN placex.rank_search < 28 THEN placex.place_id ELSE placex.parent_place_id END) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; $sSQL .= "(extratags->'place') as extra_place "; $sSQL .= "from placex where place_id in ($sPlaceIDs) "; $sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank "; @@ -269,26 +270,26 @@ if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank) { $sSQL .= " union "; - $sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,"; + $sSQL .= "select 'T' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "null as placename,"; $sSQL .= "null as ref,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= "-0.15 as importance, "; - $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; + $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_tiger.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; $sSQL .= "null as extra_place "; $sSQL .= "from location_property_tiger where place_id in ($sPlaceIDs) "; $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank "; $sSQL .= "group by place_id"; if (!$this->bDeDupe) $sSQL .= ",place_id"; $sSQL .= " union "; - $sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id,'us' as country_code,"; + $sSQL .= "select 'L' as osm_type,place_id as osm_id,'place' as class,'house' as type,null as admin_level,30 as rank_search,30 as rank_address,min(place_id) as place_id, min(parent_place_id) as parent_place_id,'us' as country_code,"; $sSQL .= "get_address_by_language(place_id, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= "null as placename,"; $sSQL .= "null as ref,"; $sSQL .= "avg(ST_X(centroid)) as lon,avg(ST_Y(centroid)) as lat, "; $sSQL .= "-0.10 as importance, "; - $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; + $sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, "; $sSQL .= "null as extra_place "; $sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) "; $sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank "; @@ -337,7 +338,7 @@ aBoundingBox: bounding Box label: short description of the object class/type (English only) name: full name (currently the same as langaddress) - foundorder: further ordering value for places with same importance + foundorder: secondary ordering for places with same importance */ function lookup() { @@ -1145,12 +1146,16 @@ $aOrder[] = "$sImportanceSQL DESC"; if (sizeof($aSearch['aFullNameAddress'])) { - $aOrder[] = '(select count(*) from (select unnest(ARRAY['.join($aSearch['aFullNameAddress'],",").']) INTERSECT select unnest(nameaddress_vector))s) DESC'; + $sExactMatchSQL = '(select count(*) from (select unnest(ARRAY['.join($aSearch['aFullNameAddress'],",").']) INTERSECT select unnest(nameaddress_vector))s) as exactmatch'; + $aOrder[] = 'exactmatch DESC'; + } else { + $sExactMatchSQL = '0::int as exactmatch'; } if (sizeof($aTerms)) { - $sSQL = "select place_id"; + $sSQL = "select place_id, "; + $sSQL .= $sExactMatchSQL; $sSQL .= " from search_name"; $sSQL .= " where ".join(' and ',$aTerms); $sSQL .= " order by ".join(', ',$aOrder); @@ -1178,6 +1183,7 @@ //if ($aViewBoxRow['in_small'] == 't') $bViewBoxMatch = 1; //else if ($aViewBoxRow['in_large'] == 't') $bViewBoxMatch = 2; $aPlaceIDs[] = $aViewBoxRow['place_id']; + $this->exactMatchCache[$aViewBoxRow['place_id']] = $aViewBoxRow['exactmatch']; } } //var_Dump($aPlaceIDs); @@ -1575,7 +1581,24 @@ $aResult['importance'] = $aResult['importance'] + ($iCountWords*0.1); // 0.1 is a completely arbitrary number but something in the range 0.1 to 0.5 would seem right $aResult['name'] = $aResult['langaddress']; - $aResult['foundorder'] = -$aResult['addressimportance']; + // secondary ordering (for results with same importance (the smaller the better): + // - approximate importance of address parts + $aResult['foundorder'] = -$aResult['addressimportance']/10; + // - number of exact matches from the query + if (isset($this->exactMatchCache[$aResult['place_id']])) + $aResult['foundorder'] -= $this->exactMatchCache[$aResult['place_id']]; + else if (isset($this->exactMatchCache[$aResult['parent_place_id']])) + $aResult['foundorder'] -= $this->exactMatchCache[$aResult['parent_place_id']]; + // - importance of the class/type + if (isset($aClassType[$aResult['class'].':'.$aResult['type']]['importance']) + && $aClassType[$aResult['class'].':'.$aResult['type']]['importance']) + { + $aResult['foundorder'] = $aResult['foundorder'] + 0.000001 * $aClassType[$aResult['class'].':'.$aResult['type']]['importance']; + } + else + { + $aResult['foundorder'] = $aResult['foundorder'] + 0.001; + } $aSearchResults[$iResNum] = $aResult; } uasort($aSearchResults, 'byImportance'); From 0687065748bac9a56fe36922fdce15e2010a50a5 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 4 Dec 2013 08:07:38 +0100 Subject: [PATCH 27/29] enable indexed_status check for all --- lib/ReverseGeocode.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ReverseGeocode.php b/lib/ReverseGeocode.php index 8503e1eb..cb8c0c6b 100644 --- a/lib/ReverseGeocode.php +++ b/lib/ReverseGeocode.php @@ -95,8 +95,8 @@ $sSQL .= ' and rank_search != 28 and rank_search >= '.$iMaxRank; $sSQL .= ' and (name is not null or housenumber is not null)'; $sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\')'; - $sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') '; $sSQL .= ' and indexed_status = 0 '; + $sSQL .= ' and (ST_GeometryType(geometry) not in (\'ST_Polygon\',\'ST_MultiPolygon\') '; $sSQL .= ' OR ST_DWithin('.$sPointSQL.', centroid, '.$fSearchDiam.'))'; $sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1'; if (CONST_Debug) var_dump($sSQL); From 10f3a4df2335b7a960a54b7606f1a4bda633fbb6 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 16 Dec 2013 00:00:06 +0100 Subject: [PATCH 28/29] merge postcodes from address objects Take the postcode from the first address object that has one tagged and use that as display postcode. --- sql/functions.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/functions.sql b/sql/functions.sql index 73aa4fee..1883d998 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -2481,7 +2481,7 @@ BEGIN CASE WHEN class = 'place' and type = 'postcode' THEN hstore('name', postcode) ELSE name END as name, class, type, admin_level, fromarea, isaddress, CASE WHEN address_place_id = for_place_id AND rank_address = 0 THEN 100 WHEN rank_address = 11 THEN 5 ELSE rank_address END as rank_address, - distance,calculated_country_code + distance,calculated_country_code,postcode from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = for_place_id and (cached_rank_address > 0 AND cached_rank_address < searchrankaddress) @@ -2496,6 +2496,9 @@ BEGIN IF searchpostcode IS NOT NULL and location.type = 'postcode' THEN location.isaddress := FALSE; END IF; + IF searchpostcode IS NULL and location.isaddress and location.type != 'postcode' and location.postcode IS NOT NULL THEN + searchpostcode := location.postcode; + END IF; IF location.rank_address = 4 AND location.isaddress THEN hadcountry := true; END IF; From 014f19ec685f83ccda9ec1d9dea7f887ddc5b41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodolphe=20Qui=C3=A9deville?= Date: Tue, 14 Jan 2014 23:50:22 +0100 Subject: [PATCH 29/29] Add some comments to help use munin plugin, add over limit detection --- munin/nominatim_importlag | 40 +++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/munin/nominatim_importlag b/munin/nominatim_importlag index 9f401311..0960ff5a 100755 --- a/munin/nominatim_importlag +++ b/munin/nominatim_importlag @@ -5,26 +5,42 @@ # Can be configured through libpq environment variables, for example # PGUSER, PGDATABASE, etc. See man page of psql for more information # -# Parameters: +# To configure munin for a default installation, add these lines to +# the file /etc/munin/plugin-conf.d/munin-node or in any file in the +# directory /etc/munin/plugin-conf.d/ +# +# [nominatim_*] +# user www-data +# env.PGUSER www-data +# env.PGPORT 5432 +# env.PGDATABASE nominatim +# env.age_warning 21600 +# env.age_critical 86400 + +# Parameters: # # config (required) # autoconf (optional - used by munin-config) # - + +. $MUNIN_LIBDIR/plugins/plugin.sh + if [ "$1" = "config" ]; then - - echo 'graph_title Data import lag' - echo 'graph_args --base 1000 -l 0' - echo 'graph_vlabel minutes' - echo 'graph_category nominatim' - echo 'age.label DB import age' - echo 'age.type GAUGE' + + echo 'graph_title Data import lag' + echo 'graph_args --base 1000 -l 0' + echo 'graph_vlabel minutes' + echo 'graph_category nominatim' + echo 'age.label DB import age' + echo 'age.type GAUGE' echo 'age.cdef age,60,/' + print_warning age + print_critical age exit 0 fi - + delay=`psql -c "copy (select extract(epoch from timezone('utc', now())-lastimportdate)::int from import_status) to stdout"` - - + + echo "age.value $delay"