diff --git a/lib/PlaceLookup.php b/lib/PlaceLookup.php index 3fd882a7..eff66fd1 100644 --- a/lib/PlaceLookup.php +++ b/lib/PlaceLookup.php @@ -138,7 +138,9 @@ class PlaceLookup // but this will never happen, because if the searched point is that close to the endnumber, the endnumber house will be directly taken from placex (in ReverseGeocode.php line 220) // and not interpolated } else { - $sSQL = "select placex.place_id, partition, osm_type, osm_id, class, type, admin_level, housenumber, street, isin, postcode, country_code, parent_place_id, linked_place_id, rank_address, rank_search, "; + $sSQL = "select placex.place_id, partition, osm_type, osm_id, class,"; + $sSQL .= " type, admin_level, housenumber, street, isin, postcode, country_code,"; + $sSQL .= " parent_place_id, linked_place_id, rank_address, rank_search, "; $sSQL .= " coalesce(importance,0.75-(rank_search::float/40)) as importance, indexed_status, indexed_date, wikipedia, calculated_country_code, "; $sSQL .= " get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress,"; $sSQL .= " get_name_by_language(name, $sLanguagePrefArraySQL) as placename,"; diff --git a/lib/init-website.php b/lib/init-website.php index 4ca927cf..967c6cce 100644 --- a/lib/init-website.php +++ b/lib/init-website.php @@ -52,7 +52,8 @@ function failInternalError($sError, $sSQL = false, $vDumpVar = false) echo "

Internal Server Error

"; echo '

Nominatim has encountered an internal error while processing your request. This is most likely because of a bug in the software.

'; echo "

Details: ".$sError,"

"; - echo '

Feel free to file an issue on Github. Please include the error message above and the URL you used.

'; + echo '

Feel free to file an issue on Github. '; + echo 'Please include the error message above and the URL you used.

'; if (CONST_Debug) { echo "

Debugging Information


"; if ($sSQL) { @@ -76,7 +77,8 @@ function userError($sError) echo "

Bad Request

"; echo '

Nominatim has encountered an error with your request.

'; echo "

Details: ".$sError."

"; - echo '

If you feel this error is incorrect feel file an issue on Github. Please include the error message above and the URL you used.

'; + echo '

If you feel this error is incorrect feel file an issue on Github. '; + echo 'Please include the error message above and the URL you used.

'; echo "\n\n"; exit; } diff --git a/lib/lib.php b/lib/lib.php index e14f52f9..de100fdc 100644 --- a/lib/lib.php +++ b/lib/lib.php @@ -494,7 +494,10 @@ function _debugDumpGroupedSearches($aData, $aTokens) } } echo ""; - echo ""; + echo ""; + echo ""; + echo ""; + echo ""; foreach ($aData as $iRank => $aRankedSet) { foreach ($aRankedSet as $aRow) { echo ""; diff --git a/phpcs.xml b/phpcs.xml index c8b8a80e..9f705d83 100644 --- a/phpcs.xml +++ b/phpcs.xml @@ -8,14 +8,10 @@ - - - - - - + + diff --git a/utils/importWikipedia.php b/utils/importWikipedia.php index 6138398b..5271d233 100755 --- a/utils/importWikipedia.php +++ b/utils/importWikipedia.php @@ -307,8 +307,14 @@ function _templatesToProperties($aTemplates) if (isset($aCMDResult['parse-wikipedia'])) { $oDB =& getDB(); - $aArticleNames = $oDB->getCol('select page_title from content where page_namespace = 0 and page_id %10 = '.$aCMDResult['parse-wikipedia'].' and (page_content ilike \'%{{Coord%\' or (page_content ilike \'%lat%\' and page_content ilike \'%lon%\'))'); - // $aArticleNames = $oDB->getCol($sSQL = 'select page_title from content where page_namespace = 0 and (page_content ilike \'%{{Coord%\' or (page_content ilike \'%lat%\' and page_content ilike \'%lon%\')) and page_title in (\'Virginia\')'); + $sSQL = 'select page_title from content where page_namespace = 0 and page_id %10 = '; + $sSQL .= $aCMDResult['parse-wikipedia']; + $sSQL .= ' and (page_content ilike \'%{{Coord%\' or (page_content ilike \'%lat%\' and page_content ilike \'%lon%\'))' + $aArticleNames = $oDB->getCol($sSQL); + /* $aArticleNames = $oDB->getCol($sSQL = 'select page_title from content where page_namespace = 0 + and (page_content ilike \'%{{Coord%\' or (page_content ilike \'%lat%\' + and page_content ilike \'%lon%\')) and page_title in (\'Virginia\')'); + */ foreach ($aArticleNames as $sArticleName) { $sPageText = $oDB->getOne('select page_content from content where page_namespace = 0 and page_title = \''.pg_escape_string($sArticleName).'\''); $aP = _templatesToProperties(_parseWikipediaContent($sPageText)); @@ -520,7 +526,10 @@ if (isset($aCMDResult['link'])) { elseif ($iRank <= 26) $fMaxDist = 0.001; else $fMaxDist = 0.001; } - echo "-- FOUND \"".substr($aNominatRecords[$i]['DISPLAY_NAME'], 0, 50)."\", ".$aNominatRecords[$i]['CLASS'].", ".$aNominatRecords[$i]['TYPE'].", ".$aNominatRecords[$i]['PLACE_RANK'].", ".$aNominatRecords[$i]['OSM_TYPE']." (dist:$fDiff, max:$fMaxDist)\n"; + echo "-- FOUND \"".substr($aNominatRecords[$i]['DISPLAY_NAME'], 0, 50); + echo "\", ".$aNominatRecords[$i]['CLASS'].", ".$aNominatRecords[$i]['TYPE']; + echo ", ".$aNominatRecords[$i]['PLACE_RANK'].", ".$aNominatRecords[$i]['OSM_TYPE']; + echo " (dist:$fDiff, max:$fMaxDist)\n"; if ($fDiff > $fMaxDist) { echo "-- Diff too big $fDiff (max: $fMaxDist)".$aRecord['lat'].','.$aNominatRecords[$i]['LAT'].' & '.$aRecord['lon'].','.$aNominatRecords[$i]['LON']." \n"; } else { diff --git a/utils/setup.php b/utils/setup.php index 9efbcf35..d08f3bd7 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -346,7 +346,11 @@ if ($aCMDResult['load-data'] || $aCMDResult['all']) { } // used by getorcreate_word_id to ignore frequent partial words - if (!pg_query($oDB->connection, 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS $$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE')) fail(pg_last_error($oDB->connection)); + $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; + $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; + if (!pg_query($oDB->connection, $sSQL)) { + fail(pg_last_error($oDB->connection)); + } echo ".\n"; // pre-create the word list diff --git a/utils/update.php b/utils/update.php index 95e99b9f..232a23c9 100755 --- a/utils/update.php +++ b/utils/update.php @@ -166,12 +166,17 @@ if ($aResult['deduplicate']) { $aPartitions = chksql($oDB->getCol($sSQL)); $aPartitions[] = 0; - $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token"; + $sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' '"; + $sSQL .= " and class is null and type is null and country_code is null"; + $sSQL .= " group by word_token having count(*) > 1 order by word_token"; $aDuplicateTokens = chksql($oDB->getAll($sSQL)); foreach ($aDuplicateTokens as $aToken) { if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue; echo "Deduping ".$aToken['word_token']."\n"; - $sSQL = "select word_id,(select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num from word where word_token = '".$aToken['word_token']."' and class is null and type is null and country_code is null order by num desc"; + $sSQL = "select word_id,"; + $sSQL .= " (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num"; + $sSQL .= " from word where word_token = '".$aToken['word_token']; + $sSQL .= "' and class is null and type is null and country_code is null order by num desc"; $aTokenSet = chksql($oDB->getAll($sSQL)); $aKeep = array_shift($aTokenSet); diff --git a/website/deletable.php b/website/deletable.php index d99eb5d1..531bdadb 100755 --- a/website/deletable.php +++ b/website/deletable.php @@ -10,7 +10,10 @@ $sOutputFormat = 'html'; $oDB =& getDB(); -$sSQL = "select placex.place_id, calculated_country_code as country_code, name->'name' as name, i.* from placex, import_polygon_delete i where placex.osm_id = i.osm_id and placex.osm_type = i.osm_type and placex.class = i.class and placex.type = i.type"; +$sSQL = "select placex.place_id, calculated_country_code as country_code,"; +$sSQL .= " name->'name' as name, i.* from placex, import_polygon_delete i"; +$sSQL .= " where placex.osm_id = i.osm_id and placex.osm_type = i.osm_type"; +$sSQL .= " and placex.class = i.class and placex.type = i.type"; $aPolygons = chksql($oDB->getAll($sSQL), "Could not get list of deleted OSM elements."); if (CONST_Debug) { diff --git a/website/details.php b/website/details.php index 9fb8a900..99cf9597 100755 --- a/website/details.php +++ b/website/details.php @@ -25,7 +25,12 @@ if ($sOsmType && $iOsmId > 0) { // Be nice about our error messages for broken geometry if (!$sPlaceId) { - $aPointDetails = chksql($oDB->getRow("select osm_type, osm_id, errormessage, class, type, get_name_by_language(name,$sLanguagePrefArraySQL) as localname, ST_AsText(prevgeometry) as prevgeom, ST_AsText(newgeometry) as newgeom from import_polygon_error where osm_type = '".$sOsmType."' and osm_id = ".$iOsmId." order by updated desc limit 1")); + $sSQL = "select osm_type, osm_id, errormessage, class, type,"; + $sSQL .= " get_name_by_language(name,$sLanguagePrefArraySQL) as localname,"; + $sSQL .= " ST_AsText(prevgeometry) as prevgeom, ST_AsText(newgeometry) as newgeom"; + $sSQL .= " from import_polygon_error where osm_type = '".$sOsmType; + $sSQL .= "' and osm_id = ".$iOsmId." order by updated desc limit 1"; + $aPointDetails = chksql($oDB->getRow($sSQL)); if (!PEAR::isError($aPointDetails) && $aPointDetails) { if (preg_match('/\[(-?\d+\.\d+) (-?\d+\.\d+)\]/', $aPointDetails['errormessage'], $aMatches)) { $aPointDetails['error_x'] = $aMatches[1]; @@ -58,8 +63,13 @@ if (CONST_Use_Aux_Location_data) { $hLog = logStart($oDB, 'details', $_SERVER['QUERY_STRING'], $aLangPrefOrder); // Get the details for this point -$sSQL = "select place_id, osm_type, osm_id, class, type, name, admin_level, housenumber, street, isin, postcode, calculated_country_code as country_code, importance, wikipedia,"; -$sSQL .= " to_char(indexed_date, 'YYYY-MM-DD HH24:MI') as indexed_date, parent_place_id, rank_address, rank_search, get_searchrank_label(rank_search) as rank_search_label, get_name_by_language(name,$sLanguagePrefArraySQL) as localname, "; +$sSQL = "select place_id, osm_type, osm_id, class, type, name, admin_level,"; +$sSQL .= " housenumber, street, isin, postcode, calculated_country_code as country_code,"; +$sSQL .= " importance, wikipedia,"; +$sSQL .= " to_char(indexed_date, 'YYYY-MM-DD HH24:MI') as indexed_date,"; +$sSQL .= " parent_place_id, rank_address, rank_search,"; +$sSQL .= " get_searchrank_label(rank_search) as rank_search_label,"; +$sSQL .= " get_name_by_language(name,$sLanguagePrefArraySQL) as localname, "; $sSQL .= " ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, "; //$sSQL .= " ST_Area(geometry::geography) as area, "; $sSQL .= " ST_y(centroid) as lat, ST_x(centroid) as lon,"; @@ -102,7 +112,9 @@ if (PEAR::isError($aPointDetails['aExtraTags'])) { // possible timeout $aAddressLines = getAddressDetails($oDB, $sLanguagePrefArraySQL, $iPlaceID, $aPointDetails['country_code'], -1, true); // Linked places -$sSQL = "select placex.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, ST_DistanceSpheroid(geometry, placegeometry, 'SPHEROID[\"WGS 84\",6378137,298.257223563, AUTHORITY[\"EPSG\",\"7030\"]]') as distance, "; +$sSQL = "select placex.place_id, osm_type, osm_id, class, type, housenumber,"; +$sSQL .= " admin_level, rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea,"; +$sSQL .= " ST_DistanceSpheroid(geometry, placegeometry, 'SPHEROID[\"WGS 84\",6378137,298.257223563, AUTHORITY[\"EPSG\",\"7030\"]]') as distance, "; $sSQL .= " get_name_by_language(name,$sLanguagePrefArraySQL) as localname, length(name::text) as namelength "; $sSQL .= " from placex, (select centroid as placegeometry from placex where place_id = $iPlaceID) as x"; $sSQL .= " where linked_place_id = $iPlaceID"; @@ -113,7 +125,9 @@ if (PEAR::isError($aLinkedLines)) { // possible timeout } // All places this is an imediate parent of -$sSQL = "select obj.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, ST_DistanceSpheroid(geometry, placegeometry, 'SPHEROID[\"WGS 84\",6378137,298.257223563, AUTHORITY[\"EPSG\",\"7030\"]]') as distance, "; +$sSQL = "select obj.place_id, osm_type, osm_id, class, type, housenumber,"; +$sSQL .= " admin_level, rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea,"; +$sSQL .= " ST_DistanceSpheroid(geometry, placegeometry, 'SPHEROID[\"WGS 84\",6378137,298.257223563, AUTHORITY[\"EPSG\",\"7030\"]]') as distance, "; $sSQL .= " get_name_by_language(name,$sLanguagePrefArraySQL) as localname, length(name::text) as namelength "; $sSQL .= " from (select placex.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, rank_search, geometry, name from placex "; $sSQL .= " where parent_place_id = $iPlaceID order by rank_address asc,rank_search asc limit 500) as obj,"; diff --git a/website/hierarchy.php b/website/hierarchy.php index 457f37ca..131a9ca1 100755 --- a/website/hierarchy.php +++ b/website/hierarchy.php @@ -25,7 +25,12 @@ if ($sOsmType && $iOsmId > 0) { // Be nice about our error messages for broken geometry if (!$sPlaceId) { - $aPointDetails = chksql($oDB->getRow("select osm_type, osm_id, errormessage, class, type, get_name_by_language(name,$sLanguagePrefArraySQL) as localname, ST_AsText(prevgeometry) as prevgeom, ST_AsText(newgeometry) as newgeom from import_polygon_error where osm_type = '".$sOsmType."' and osm_id = ".$iOsmId." order by updated desc limit 1")); + $sSQL = "select osm_type, osm_id, errormessage, class, type,"; + $sSQL .= " get_name_by_language(name,$sLanguagePrefArraySQL) as localname,"; + $sSQL .= " ST_AsText(prevgeometry) as prevgeom, ST_AsText(newgeometry) as newgeom"; + $sSQL .= " from import_polygon_error where osm_type = '".$sOsmType; + $sSQL .= "' and osm_id = ".$iOsmId." order by updated desc limit 1"; + $aPointDetails = chksql($oDB->getRow($sSQL)); if ($aPointDetails) { if (preg_match('/\[(-?\d+\.\d+) (-?\d+\.\d+)\]/', $aPointDetails['errormessage'], $aMatches)) { $aPointDetails['error_x'] = $aMatches[1]; @@ -87,7 +92,8 @@ if ($sOutputFormat == 'json') { $aRelatedPlaceIDs = chksql($oDB->getCol($sSQL = "select place_id from placex where linked_place_id = $iPlaceID or place_id = $iPlaceID")); -$sSQL = "select obj.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, st_area(geometry) as area, "; +$sSQL = "select obj.place_id, osm_type, osm_id, class, type, housenumber, admin_level,"; +$sSQL .= " rank_address, ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea, st_area(geometry) as area, "; $sSQL .= " get_name_by_language(name,$sLanguagePrefArraySQL) as localname, length(name::text) as namelength "; $sSQL .= " from (select placex.place_id, osm_type, osm_id, class, type, housenumber, admin_level, rank_address, rank_search, geometry, name from placex "; $sSQL .= " where parent_place_id in (".join(',', $aRelatedPlaceIDs).") and name is not null order by rank_address asc,rank_search asc limit 500) as obj"; diff --git a/wikidata/parse.php b/wikidata/parse.php index 11b30fe1..d8981c6c 100755 --- a/wikidata/parse.php +++ b/wikidata/parse.php @@ -115,8 +115,21 @@ if ($hFile) { } elseif ($aClaim['m'][2] == 'globecoordinate') { // if ($aClaim['m'][3]['globe'] != 'http://www.wikidata.org/entity/Q2') $bFail = true; - fputcsv($hFileEntityProperty, array($iID, $iClaim, $iPID, null, null, "SRID=4326;POINT(".((float) $aClaim['m'][3]['longitude'])." ".((float)$aClaim['m'][3]['latitude']).")", null)); - // echo "insert into entity_property values (nextval('seq_entity_property'),".$iID.",".$iPID.",null,null,ST_SetSRID(ST_MakePoint(".((float)$aClaim['m'][3]['longitude']).", ".((float)$aClaim['m'][3]['latitude'])."),4326));\n"; + fputcsv( + $hFileEntityProperty, + array( + $iID, + $iClaim, + $iPID, + null, + null, + "SRID=4326;POINT(".((float) $aClaim['m'][3]['longitude'])." ".((float)$aClaim['m'][3]['latitude']).")", null + ) + ); + /* echo "insert into entity_property values (nextval('seq_entity_property'),"; + * echo $iID.",".$iPID.",null,null,ST_SetSRID(ST_MakePoint(".((float)$aClaim['m'][3]['longitude']); + * echo ", ".((float)$aClaim['m'][3]['latitude'])."),4326));\n"; + */ } elseif ($aClaim['m'][2] == 'time') { // TODO! /* diff --git a/wikidata/pull_wikipedia_logs.php b/wikidata/pull_wikipedia_logs.php index a7f6a9a3..bdaab52a 100644 --- a/wikidata/pull_wikipedia_logs.php +++ b/wikidata/pull_wikipedia_logs.php @@ -71,7 +71,9 @@ for ($iTimestamp = mktime(0, 0, 0, 5, 1, 2013); $iTimestamp < mktime(0, 0, 0, 6, // Notes: /* - gzip -dc $FILE.gz | grep -e "^en [^ :]\+ [0-9]\+" | sed "s#\(^[a-z]\{2\}\) \([^ :]\+\) \([0-9]\+\) [0-9]\+#update wikipedia_article set hit_count = coalesce(hit_count,0) + \3 where language = '\1' and title = catch_decode_url_part('\2');#g" | /opt/mapquest/stdbase-dev$ + gzip -dc $FILE.gz | grep -e "^en [^ :]\+ [0-9]\+" | + sed "s#\(^[a-z]\{2\}\) \([^ :]\+\) \([0-9]\+\) [0-9]\+#update wikipedia_article set hit_count = coalesce(hit_count,0) + \3 where language = '\1' + and title = catch_decode_url_part('\2');#g" | /opt/mapquest/stdbase-dev$ cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = s,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g" cat totals.txt | sed "s#\(^[a-z]\{2\}\) \([^ ]\+\) \([0-9]\+\)\$#update entity_link set hits = coalesce(hits,0) + \3 where target = '\1wiki' and value = catch_decode_url_part('\2');#g" */
rankName TokensName NotAddress TokensAddress Notcountryoperatorclasstypehouse#LatLonRadius
rankName TokensName NotAddress TokensAddress Notcountryoperatorclasstypehouse#LatLonRadius