mirror of
https://github.com/osm-search/Nominatim.git
synced 2026-02-16 15:47:58 +00:00
Merge remote-tracking branch 'origin/master' into osmline
Conflicts: lib/Geocode.php lib/PlaceLookup.php sql/functions.sql sql/tables.sql utils/setup.php
This commit is contained in:
2930
data/us_state.sql
2930
data/us_state.sql
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
132
lib/Geocode.php
132
lib/Geocode.php
@@ -228,11 +228,9 @@
|
|||||||
function loadParamArray($aParams)
|
function loadParamArray($aParams)
|
||||||
{
|
{
|
||||||
if (isset($aParams['addressdetails'])) $this->bIncludeAddressDetails = (bool)$aParams['addressdetails'];
|
if (isset($aParams['addressdetails'])) $this->bIncludeAddressDetails = (bool)$aParams['addressdetails'];
|
||||||
if ((float) CONST_Postgresql_Version > 9.2)
|
if (isset($aParams['extratags'])) $this->bIncludeExtraTags = (bool)$aParams['extratags'];
|
||||||
{
|
if (isset($aParams['namedetails'])) $this->bIncludeNameDetails = (bool)$aParams['namedetails'];
|
||||||
if (isset($aParams['extratags'])) $this->bIncludeExtraTags = (bool)$aParams['extratags'];
|
|
||||||
if (isset($aParams['namedetails'])) $this->bIncludeNameDetails = (bool)$aParams['namedetails'];
|
|
||||||
}
|
|
||||||
if (isset($aParams['bounded'])) $this->bBoundedSearch = (bool)$aParams['bounded'];
|
if (isset($aParams['bounded'])) $this->bBoundedSearch = (bool)$aParams['bounded'];
|
||||||
if (isset($aParams['dedupe'])) $this->bDeDupe = (bool)$aParams['dedupe'];
|
if (isset($aParams['dedupe'])) $this->bDeDupe = (bool)$aParams['dedupe'];
|
||||||
|
|
||||||
@@ -433,18 +431,41 @@
|
|||||||
|
|
||||||
if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank)
|
if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank)
|
||||||
{
|
{
|
||||||
//query also location_property_osmline and location_property_tiger and location_property_aux
|
if (CONST_Use_US_Tiger_Data)
|
||||||
//Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
|
|
||||||
//only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
|
|
||||||
$sHousenumbers = "";
|
|
||||||
$i = 0;
|
|
||||||
$length = count($aPlaceIDs);
|
|
||||||
foreach($aPlaceIDs as $placeID => $housenumber)
|
|
||||||
{
|
{
|
||||||
$i++;
|
//query also location_property_tiger and location_property_aux
|
||||||
$sHousenumbers .= "(".$placeID.", ".$housenumber.")";
|
//Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
|
||||||
if($i<$length)
|
//only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
|
||||||
$sHousenumbers .= ", ";
|
$sHousenumbers = "";
|
||||||
|
$i = 0;
|
||||||
|
$length = count($aPlaceIDs);
|
||||||
|
foreach($aPlaceIDs as $placeID => $housenumber)
|
||||||
|
{
|
||||||
|
$i++;
|
||||||
|
$sHousenumbers .= "(".$placeID.", ".$housenumber.")";
|
||||||
|
if($i<$length)
|
||||||
|
$sHousenumbers .= ", ";
|
||||||
|
}
|
||||||
|
|
||||||
|
$sSQL .= "union ";
|
||||||
|
$sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
|
||||||
|
$sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
|
||||||
|
$sSQL .= ", null as placename";
|
||||||
|
$sSQL .= ", null as ref";
|
||||||
|
if ($this->bIncludeExtraTags) $sSQL .= ", null as extra";
|
||||||
|
if ($this->bIncludeNameDetails) $sSQL .= ", null as names";
|
||||||
|
$sSQL .= ", avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
|
||||||
|
$sSQL .= $sImportanceSQL."-1.15 as importance ";
|
||||||
|
$sSQL .= ", (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";
|
||||||
|
$sSQL .= ", null as extra_place ";
|
||||||
|
$sSQL .= " from (select place_id";
|
||||||
|
//interpolate the Tiger housenumbers here
|
||||||
|
$sSQL .= ", ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) as centroid, parent_place_id, housenumber_for_place ";
|
||||||
|
$sSQL .= "from (location_property_tiger ";
|
||||||
|
$sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
|
||||||
|
$sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
|
||||||
|
$sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
|
||||||
|
if (!$this->bDeDupe) $sSQL .= ", place_id ";
|
||||||
}
|
}
|
||||||
// osmline, osm_type is 'I' for Interpolation Line
|
// osmline, osm_type is 'I' for Interpolation Line
|
||||||
$sSQL .= " union ";
|
$sSQL .= " union ";
|
||||||
@@ -469,46 +490,25 @@
|
|||||||
$sSQL .= " group by place_id, housenumber_for_place, calculated_country_code "; //is this group by really needed?, place_id + housenumber (in combination) are unique
|
$sSQL .= " group by place_id, housenumber_for_place, calculated_country_code "; //is this group by really needed?, place_id + housenumber (in combination) are unique
|
||||||
if (!$this->bDeDupe) $sSQL .= ", place_id ";
|
if (!$this->bDeDupe) $sSQL .= ", place_id ";
|
||||||
|
|
||||||
// tiger
|
if (CONST_Use_Aux_Location_data)
|
||||||
$sSQL .= " union ";
|
{
|
||||||
$sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
|
$sSQL .= " union ";
|
||||||
$sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
|
$sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
|
||||||
$sSQL .= ", null as placename";
|
$sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
|
||||||
$sSQL .= ", null as ref";
|
$sSQL .= "null as placename, ";
|
||||||
if ($this->bIncludeExtraTags) $sSQL .= ", null as extra";
|
$sSQL .= "null as ref, ";
|
||||||
if ($this->bIncludeNameDetails) $sSQL .= ", null as names";
|
if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
|
||||||
$sSQL .= ", avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
|
if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
|
||||||
$sSQL .= $sImportanceSQL."-1.15 as importance ";
|
$sSQL .= "avg(ST_X(centroid)) as lon, avg(ST_Y(centroid)) as lat, ";
|
||||||
$sSQL .= ", (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance ";
|
$sSQL .= $sImportanceSQL."-1.10 as importance, ";
|
||||||
$sSQL .= ", null as extra_place ";
|
$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
|
||||||
$sSQL .= " from (select place_id";
|
$sSQL .= "null as extra_place ";
|
||||||
//interpolate the Tiger housenumbers here
|
$sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
|
||||||
$sSQL .= ", CASE WHEN startnumber != endnumber THEN ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) ";
|
$sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
|
||||||
$sSQL .= " ELSE ST_LineInterpolatePoint(linegeo, 0.5) END as centroid";
|
$sSQL .= "group by place_id";
|
||||||
$sSQL .= ", parent_place_id, housenumber_for_place ";
|
if (!$this->bDeDupe) $sSQL .= ", place_id";
|
||||||
$sSQL .= " from (location_property_tiger ";
|
$sSQL .= ", get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) ";
|
||||||
$sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
|
}
|
||||||
$sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
|
|
||||||
$sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
|
|
||||||
if (!$this->bDeDupe) $sSQL .= ", place_id ";
|
|
||||||
|
|
||||||
// aux
|
|
||||||
$sSQL .= " union ";
|
|
||||||
$sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
|
|
||||||
$sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
|
|
||||||
$sSQL .= "null as placename, ";
|
|
||||||
$sSQL .= "null as ref, ";
|
|
||||||
if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
|
|
||||||
if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
|
|
||||||
$sSQL .= "avg(ST_X(centroid)) as lon, avg(ST_Y(centroid)) as lat, ";
|
|
||||||
$sSQL .= $sImportanceSQL."-1.10 as importance, ";
|
|
||||||
$sSQL .= "(select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(location_property_aux.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
|
|
||||||
$sSQL .= "null as extra_place ";
|
|
||||||
$sSQL .= "from location_property_aux where place_id in ($sPlaceIDs) ";
|
|
||||||
$sSQL .= "and 30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
|
|
||||||
$sSQL .= "group by place_id";
|
|
||||||
if (!$this->bDeDupe) $sSQL .= ", place_id";
|
|
||||||
$sSQL .= ", get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) ";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
$sSQL .= " order by importance desc";
|
$sSQL .= " order by importance desc";
|
||||||
@@ -1487,7 +1487,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
// If nothing found try the aux fallback table
|
// If nothing found try the aux fallback table
|
||||||
if (!sizeof($aPlaceIDs))
|
if (CONST_Use_Aux_Location_data && !sizeof($aPlaceIDs))
|
||||||
{
|
{
|
||||||
$sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
|
$sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
|
||||||
if (sizeof($this->aExcludePlaceIDs))
|
if (sizeof($this->aExcludePlaceIDs))
|
||||||
@@ -1498,8 +1498,10 @@
|
|||||||
if (CONST_Debug) var_dump($sSQL);
|
if (CONST_Debug) var_dump($sSQL);
|
||||||
$aPlaceIDs = $this->oDB->getCol($sSQL);
|
$aPlaceIDs = $this->oDB->getCol($sSQL);
|
||||||
}
|
}
|
||||||
//if nothing found search in Tiger data for this housenumber(location_property_tiger)
|
|
||||||
if (!sizeof($aPlaceIDs))
|
//if nothing was found in placex or location_property_aux, then search in Tiger data for this housenumber(location_property_tiger)
|
||||||
|
$searchedHousenumber = intval($aSearch['sHouseNumber']);
|
||||||
|
if (CONST_Use_US_Tiger_Data && !sizeof($aPlaceIDs))
|
||||||
{
|
{
|
||||||
//new query for lines, not housenumbers anymore
|
//new query for lines, not housenumbers anymore
|
||||||
if($searchedHousenumber%2 == 0){
|
if($searchedHousenumber%2 == 0){
|
||||||
@@ -1672,11 +1674,13 @@
|
|||||||
$sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank ";
|
$sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank ";
|
||||||
if (14 >= $this->iMinAddressRank && 14 <= $this->iMaxAddressRank) $sSQL .= " OR (extratags->'place') = 'city'";
|
if (14 >= $this->iMinAddressRank && 14 <= $this->iMaxAddressRank) $sSQL .= " OR (extratags->'place') = 'city'";
|
||||||
if ($this->aAddressRankList) $sSQL .= " OR placex.rank_address in (".join(',',$this->aAddressRankList).")";
|
if ($this->aAddressRankList) $sSQL .= " OR placex.rank_address in (".join(',',$this->aAddressRankList).")";
|
||||||
$sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
|
if (CONST_Use_US_Tiger_Data)
|
||||||
$sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
|
{
|
||||||
if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
|
$sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
|
||||||
$sSQL .= ") UNION select place_id from location_property_osmline where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
|
$sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
|
||||||
$sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank)";
|
if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
|
||||||
|
}
|
||||||
|
$sSQL .= ")";
|
||||||
if (CONST_Debug) var_dump($sSQL);
|
if (CONST_Debug) var_dump($sSQL);
|
||||||
$aFilteredPlaceIDs = $this->oDB->getCol($sSQL);
|
$aFilteredPlaceIDs = $this->oDB->getCol($sSQL);
|
||||||
$tempIDs = array();
|
$tempIDs = array();
|
||||||
|
|||||||
@@ -42,18 +42,12 @@
|
|||||||
|
|
||||||
function setIncludeExtraTags($bExtraTags = false)
|
function setIncludeExtraTags($bExtraTags = false)
|
||||||
{
|
{
|
||||||
if ((float) CONST_Postgresql_Version > 9.2)
|
$this->bExtraTags = $bExtraTags;
|
||||||
{
|
|
||||||
$this->bExtraTags = $bExtraTags;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function setIncludeNameDetails($bNameDetails = false)
|
function setIncludeNameDetails($bNameDetails = false)
|
||||||
{
|
{
|
||||||
if ((float) CONST_Postgresql_Version > 9.2)
|
$this->bNameDetails = $bNameDetails;
|
||||||
{
|
|
||||||
$this->bNameDetails = $bNameDetails;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -128,7 +122,7 @@
|
|||||||
|
|
||||||
$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
|
$sLanguagePrefArraySQL = "ARRAY[".join(',',array_map("getDBQuoted", $this->aLangPrefOrder))."]";
|
||||||
|
|
||||||
if ($this->sType == 'tiger')
|
if (CONST_Use_US_Tiger_Data && $this->sType == 'tiger')
|
||||||
{
|
{
|
||||||
$sSQL = "select place_id,partition, 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, housenumber, null as street, null as isin, postcode,";
|
$sSQL = "select place_id,partition, 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, housenumber, null as street, null as isin, postcode,";
|
||||||
$sSQL .= " 'us' as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
|
$sSQL .= " 'us' as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
|
||||||
@@ -193,7 +187,7 @@
|
|||||||
|
|
||||||
if ($this->bAddressDetails)
|
if ($this->bAddressDetails)
|
||||||
{
|
{
|
||||||
if ($this->sType == 'tiger' || $this->sType == 'interpolation') // to get addressdetails for interpolation lines and tiger data, the housenumber is needed
|
if(CONST_Use_US_Tiger_Data && $this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed
|
||||||
$aAddress = $this->getAddressNames($aPlace['housenumber']);
|
$aAddress = $this->getAddressNames($aPlace['housenumber']);
|
||||||
else
|
else
|
||||||
$aAddress = $this->getAddressNames();
|
$aAddress = $this->getAddressNames();
|
||||||
|
|||||||
@@ -259,7 +259,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Only street found? If it's in the US we can check TIGER data for nearest housenumber
|
// Only street found? If it's in the US we can check TIGER data for nearest housenumber
|
||||||
if ($bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
|
if (CONST_Use_US_Tiger_Data && $bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
|
||||||
{
|
{
|
||||||
$fSearchDiam = 0.001;
|
$fSearchDiam = 0.001;
|
||||||
$sSQL = 'SELECT place_id,parent_place_id,30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction';
|
$sSQL = 'SELECT place_id,parent_place_id,30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction';
|
||||||
|
|||||||
13
lib/db.php
13
lib/db.php
@@ -24,3 +24,16 @@
|
|||||||
return "'".pg_escape_string($s)."'";
|
return "'".pg_escape_string($s)."'";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getPostgresVersion(&$oDB)
|
||||||
|
{
|
||||||
|
$sVersionString = $oDB->getOne('select version()');
|
||||||
|
preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[^0-9]#', $sVersionString, $aMatches);
|
||||||
|
return (float) ($aMatches[1].'.'.$aMatches[2]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getPostgisVersion(&$oDB)
|
||||||
|
{
|
||||||
|
$sVersionString = $oDB->getOne('select postgis_full_version()');
|
||||||
|
preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
|
||||||
|
return (float) ($aMatches[1].'.'.$aMatches[2]);
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,12 +12,6 @@
|
|||||||
}
|
}
|
||||||
if ($_SERVER['REQUEST_METHOD'] == 'OPTIONS') exit;
|
if ($_SERVER['REQUEST_METHOD'] == 'OPTIONS') exit;
|
||||||
|
|
||||||
if (CONST_ClosedForIndexing && strpos(CONST_ClosedForIndexingExceptionIPs, ','.$_SERVER["REMOTE_ADDR"].',') === false)
|
|
||||||
{
|
|
||||||
echo "Closed for re-indexing...";
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
$aBucketKeys = array();
|
$aBucketKeys = array();
|
||||||
|
|
||||||
if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST)));
|
if (isset($_SERVER["HTTP_REFERER"])) $aBucketKeys[] = str_replace('www.','',strtolower(parse_url($_SERVER["HTTP_REFERER"], PHP_URL_HOST)));
|
||||||
|
|||||||
112
lib/lib.php
112
lib/lib.php
@@ -55,17 +55,6 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function getBlockingProcesses()
|
|
||||||
{
|
|
||||||
$sStats = file_get_contents('/proc/stat');
|
|
||||||
if (preg_match('/procs_blocked ([0-9]+)/i', $sStats, $aMatches))
|
|
||||||
{
|
|
||||||
return (int)$aMatches[1];
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function getLoadAverage()
|
function getLoadAverage()
|
||||||
{
|
{
|
||||||
$sLoadAverage = file_get_contents('/proc/loadavg');
|
$sLoadAverage = file_get_contents('/proc/loadavg');
|
||||||
@@ -110,16 +99,7 @@
|
|||||||
{
|
{
|
||||||
if ($a['importance'] != $b['importance'])
|
if ($a['importance'] != $b['importance'])
|
||||||
return ($a['importance'] > $b['importance']?-1:1);
|
return ($a['importance'] > $b['importance']?-1:1);
|
||||||
/*
|
|
||||||
if ($a['aPointPolygon']['numfeatures'] != $b['aPointPolygon']['numfeatures'])
|
|
||||||
return ($a['aPointPolygon']['numfeatures'] > $b['aPointPolygon']['numfeatures']?-1:1);
|
|
||||||
if ($a['aPointPolygon']['area'] != $b['aPointPolygon']['area'])
|
|
||||||
return ($a['aPointPolygon']['area'] > $b['aPointPolygon']['area']?-1:1);
|
|
||||||
// if ($a['levenshtein'] != $b['levenshtein'])
|
|
||||||
// return ($a['levenshtein'] < $b['levenshtein']?-1:1);
|
|
||||||
if ($a['rank_search'] != $b['rank_search'])
|
|
||||||
return ($a['rank_search'] < $b['rank_search']?-1:1);
|
|
||||||
*/
|
|
||||||
return ($a['foundorder'] < $b['foundorder']?-1:1);
|
return ($a['foundorder'] < $b['foundorder']?-1:1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,7 +209,6 @@
|
|||||||
{
|
{
|
||||||
$aTokens[' '.$sWord] = ' '.$sWord;
|
$aTokens[' '.$sWord] = ' '.$sWord;
|
||||||
$aTokens[$sWord] = $sWord;
|
$aTokens[$sWord] = $sWord;
|
||||||
//if (!strpos($sWord,' ')) $aTokens[$sWord] = $sWord;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return $aTokens;
|
return $aTokens;
|
||||||
@@ -240,34 +219,6 @@
|
|||||||
GB Postcode functions
|
GB Postcode functions
|
||||||
*/
|
*/
|
||||||
|
|
||||||
function gbPostcodeAlphaDifference($s1, $s2)
|
|
||||||
{
|
|
||||||
$aValues = array(
|
|
||||||
'A'=>0,
|
|
||||||
'B'=>1,
|
|
||||||
'D'=>2,
|
|
||||||
'E'=>3,
|
|
||||||
'F'=>4,
|
|
||||||
'G'=>5,
|
|
||||||
'H'=>6,
|
|
||||||
'J'=>7,
|
|
||||||
'L'=>8,
|
|
||||||
'N'=>9,
|
|
||||||
'O'=>10,
|
|
||||||
'P'=>11,
|
|
||||||
'Q'=>12,
|
|
||||||
'R'=>13,
|
|
||||||
'S'=>14,
|
|
||||||
'T'=>15,
|
|
||||||
'U'=>16,
|
|
||||||
'W'=>17,
|
|
||||||
'X'=>18,
|
|
||||||
'Y'=>19,
|
|
||||||
'Z'=>20);
|
|
||||||
return abs(($aValues[$s1[0]]*21+$aValues[$s1[1]]) - ($aValues[$s2[0]]*21+$aValues[$s2[1]]));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function gbPostcodeCalculate($sPostcode, $sPostcodeSector, $sPostcodeEnd, &$oDB)
|
function gbPostcodeCalculate($sPostcode, $sPostcodeSector, $sPostcodeEnd, &$oDB)
|
||||||
{
|
{
|
||||||
// Try an exact match on the gb_postcode table
|
// Try an exact match on the gb_postcode table
|
||||||
@@ -294,66 +245,6 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function usPostcodeCalculate($sPostcode, &$oDB)
|
|
||||||
{
|
|
||||||
$iZipcode = (int)$sPostcode;
|
|
||||||
|
|
||||||
// Try an exact match on the us_zippostcode table
|
|
||||||
$sSQL = 'select zipcode, ST_X(ST_Centroid(geometry)) as lon,ST_Y(ST_Centroid(geometry)) as lat from us_zipcode where zipcode = '.$iZipcode;
|
|
||||||
$aNearPostcodes = $oDB->getAll($sSQL);
|
|
||||||
if (PEAR::IsError($aNearPostcodes))
|
|
||||||
{
|
|
||||||
var_dump($sSQL, $aNearPostcodes);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!sizeof($aNearPostcodes))
|
|
||||||
{
|
|
||||||
$sSQL = 'select zipcode,ST_X(ST_Centroid(geometry)) as lon,ST_Y(ST_Centroid(geometry)) as lat from us_zipcode where zipcode between '.($iZipcode-100).' and '.($iZipcode+100).' order by abs(zipcode - '.$iZipcode.') asc limit 5';
|
|
||||||
$aNearPostcodes = $oDB->getAll($sSQL);
|
|
||||||
if (PEAR::IsError($aNearPostcodes))
|
|
||||||
{
|
|
||||||
var_dump($sSQL, $aNearPostcodes);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!sizeof($aNearPostcodes))
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
$fTotalLat = 0;
|
|
||||||
$fTotalLon = 0;
|
|
||||||
$fTotalFac = 0;
|
|
||||||
foreach($aNearPostcodes as $aPostcode)
|
|
||||||
{
|
|
||||||
$iDiff = abs($aPostcode['zipcode'] - $iZipcode) + 1;
|
|
||||||
if ($iDiff == 0)
|
|
||||||
$fFac = 1;
|
|
||||||
else
|
|
||||||
$fFac = 1/($iDiff*$iDiff);
|
|
||||||
|
|
||||||
$fTotalFac += $fFac;
|
|
||||||
$fTotalLat += $aPostcode['lat'] * $fFac;
|
|
||||||
$fTotalLon += $aPostcode['lon'] * $fFac;
|
|
||||||
}
|
|
||||||
if ($fTotalFac)
|
|
||||||
{
|
|
||||||
$fLat = $fTotalLat / $fTotalFac;
|
|
||||||
$fLon = $fTotalLon / $fTotalFac;
|
|
||||||
return array(array('lat' => $fLat, 'lon' => $fLon, 'radius' => 0.2));
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
|
|
||||||
/*
|
|
||||||
$fTotalFac is a surprisingly good indicator of accuracy
|
|
||||||
$iZoom = 18 + round(log($fTotalFac,32));
|
|
||||||
$iZoom = max(13,min(18,$iZoom));
|
|
||||||
*/
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
function getClassTypes()
|
function getClassTypes()
|
||||||
{
|
{
|
||||||
return array(
|
return array(
|
||||||
@@ -701,7 +592,6 @@
|
|||||||
|
|
||||||
function javascript_renderData($xVal, $iOptions = 0)
|
function javascript_renderData($xVal, $iOptions = 0)
|
||||||
{
|
{
|
||||||
header("Access-Control-Allow-Origin: *");
|
|
||||||
if (defined('PHP_VERSION_ID') && PHP_VERSION_ID > 50400)
|
if (defined('PHP_VERSION_ID') && PHP_VERSION_ID > 50400)
|
||||||
$iOptions |= JSON_UNESCAPED_UNICODE;
|
$iOptions |= JSON_UNESCAPED_UNICODE;
|
||||||
$jsonout = json_encode($xVal, $iOptions);
|
$jsonout = json_encode($xVal, $iOptions);
|
||||||
|
|||||||
82
lib/log.php
82
lib/log.php
@@ -2,68 +2,59 @@
|
|||||||
|
|
||||||
function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array())
|
function logStart(&$oDB, $sType = '', $sQuery = '', $aLanguageList = array())
|
||||||
{
|
{
|
||||||
$aStartTime = explode('.',microtime(true));
|
$fStartTime = microtime(true);
|
||||||
|
$aStartTime = explode('.', $fStartTime);
|
||||||
if (!isset($aStartTime[1])) $aStartTime[1] = '0';
|
if (!isset($aStartTime[1])) $aStartTime[1] = '0';
|
||||||
|
|
||||||
$sOutputFormat = '';
|
$sOutputFormat = '';
|
||||||
if (isset($_GET['format'])) $sOutputFormat = $_GET['format'];
|
if (isset($_GET['format'])) $sOutputFormat = $_GET['format'];
|
||||||
|
|
||||||
|
if ($sType == 'reverse')
|
||||||
|
{
|
||||||
|
$sOutQuery = (isset($_GET['lat'])?$_GET['lat']:'').'/';
|
||||||
|
if (isset($_GET['lon'])) $sOutQuery .= $_GET['lon'];
|
||||||
|
if (isset($_GET['zoom'])) $sOutQuery .= '/'.$_GET['zoom'];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
$sOutQuery = $sQuery;
|
||||||
|
|
||||||
$hLog = array(
|
$hLog = array(
|
||||||
date('Y-m-d H:i:s',$aStartTime[0]).'.'.$aStartTime[1],
|
date('Y-m-d H:i:s',$aStartTime[0]).'.'.$aStartTime[1],
|
||||||
$_SERVER["REMOTE_ADDR"],
|
$_SERVER["REMOTE_ADDR"],
|
||||||
$_SERVER['QUERY_STRING'],
|
$_SERVER['QUERY_STRING'],
|
||||||
$sQuery
|
$sOutQuery,
|
||||||
|
$sType,
|
||||||
|
$fStartTime
|
||||||
);
|
);
|
||||||
|
|
||||||
if (CONST_Log_DB)
|
if (CONST_Log_DB)
|
||||||
{
|
{
|
||||||
// Log
|
if (isset($_GET['email']))
|
||||||
if ($sType == 'search')
|
$sUserAgent = $_GET['email'];
|
||||||
{
|
elseif (isset($_SERVER['HTTP_REFERER']))
|
||||||
$oDB->query('insert into query_log values ('.getDBQuoted($hLog[0]).','.getDBQuoted($hLog[3]).','.getDBQuoted($hLog[1]).')');
|
$sUserAgent = $_SERVER['HTTP_REFERER'];
|
||||||
}
|
elseif (isset($_SERVER['HTTP_USER_AGENT']))
|
||||||
|
$sUserAgent = $_SERVER['HTTP_USER_AGENT'];
|
||||||
$sSQL = 'insert into new_query_log (type,starttime,query,ipaddress,useragent,language,format)';
|
else
|
||||||
|
$sUserAgent = '';
|
||||||
|
$sSQL = 'insert into new_query_log (type,starttime,query,ipaddress,useragent,language,format,searchterm)';
|
||||||
$sSQL .= ' values ('.getDBQuoted($sType).','.getDBQuoted($hLog[0]).','.getDBQuoted($hLog[2]);
|
$sSQL .= ' values ('.getDBQuoted($sType).','.getDBQuoted($hLog[0]).','.getDBQuoted($hLog[2]);
|
||||||
$sSQL .= ','.getDBQuoted($hLog[1]).','.getDBQuoted($_SERVER['HTTP_USER_AGENT']).','.getDBQuoted(join(',',$aLanguageList)).','.getDBQuoted($sOutputFormat).')';
|
$sSQL .= ','.getDBQuoted($hLog[1]).','.getDBQuoted($sUserAgent).','.getDBQuoted(join(',',$aLanguageList)).','.getDBQuoted($sOutputFormat).','.getDBQuoted($hLog[3]).')';
|
||||||
$oDB->query($sSQL);
|
$oDB->query($sSQL);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (CONST_Log_File && CONST_Log_File_ReverseLog != '')
|
|
||||||
{
|
|
||||||
if ($sType == 'reverse')
|
|
||||||
{
|
|
||||||
$aStartTime = explode('.',$hLog[0]);
|
|
||||||
file_put_contents(CONST_Log_File_ReverseLog,
|
|
||||||
$aStartTime[0].','.$aStartTime[1].','.
|
|
||||||
php_uname('n').','.
|
|
||||||
'"'.addslashes(isset($_SERVER['HTTP_REFERER'])?$_SERVER['HTTP_REFERER']:'').'",'.
|
|
||||||
'"'.addslashes($hLog[1]).'",'.
|
|
||||||
$_GET['lat'].','.
|
|
||||||
$_GET['lon'].','.
|
|
||||||
$_GET['zoom'].','.
|
|
||||||
'"'.addslashes($_SERVER['HTTP_USER_AGENT']).'",'.
|
|
||||||
'"'.addslashes($sOutputFormat).'"'."\n",
|
|
||||||
FILE_APPEND);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $hLog;
|
return $hLog;
|
||||||
}
|
}
|
||||||
|
|
||||||
function logEnd(&$oDB, $hLog, $iNumResults)
|
function logEnd(&$oDB, $hLog, $iNumResults)
|
||||||
{
|
{
|
||||||
$aEndTime = explode('.',microtime(true));
|
$fEndTime = microtime(true);
|
||||||
if (!$aEndTime[1]) $aEndTime[1] = '0';
|
|
||||||
$sEndTime = date('Y-m-d H:i:s',$aEndTime[0]).'.'.$aEndTime[1];
|
|
||||||
|
|
||||||
if (CONST_Log_DB)
|
if (CONST_Log_DB)
|
||||||
{
|
{
|
||||||
$sSQL = 'update query_log set endtime = '.getDBQuoted($sEndTime).', results = '.$iNumResults;
|
$aEndTime = explode('.', $fEndTime);
|
||||||
$sSQL .= ' where starttime = '.getDBQuoted($hLog[0]);
|
if (!$aEndTime[1]) $aEndTime[1] = '0';
|
||||||
$sSQL .= ' and ipaddress = '.getDBQuoted($hLog[1]);
|
$sEndTime = date('Y-m-d H:i:s',$aEndTime[0]).'.'.$aEndTime[1];
|
||||||
$sSQL .= ' and query = '.getDBQuoted($hLog[3]);
|
|
||||||
$oDB->query($sSQL);
|
|
||||||
|
|
||||||
$sSQL = 'update new_query_log set endtime = '.getDBQuoted($sEndTime).', results = '.$iNumResults;
|
$sSQL = 'update new_query_log set endtime = '.getDBQuoted($sEndTime).', results = '.$iNumResults;
|
||||||
$sSQL .= ' where starttime = '.getDBQuoted($hLog[0]);
|
$sSQL .= ' where starttime = '.getDBQuoted($hLog[0]);
|
||||||
@@ -72,19 +63,12 @@
|
|||||||
$oDB->query($sSQL);
|
$oDB->query($sSQL);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (CONST_Log_File && CONST_Log_File_SearchLog != '')
|
if (CONST_Log_File)
|
||||||
{
|
{
|
||||||
$aStartTime = explode('.',$hLog[0]);
|
$aOutdata = sprintf("[%s] %.4f %d %s \"%s\"\n",
|
||||||
file_put_contents(CONST_Log_File_SearchLog,
|
$hLog[0], $fEndTime-$hLog[5], $iNumResults,
|
||||||
$aStartTime[0].','.$aStartTime[1].','.
|
$hLog[4], $hLog[2]);
|
||||||
php_uname('n').','.
|
file_put_contents(CONST_Log_File, $aOutdata, FILE_APPEND | LOCK_EX);
|
||||||
'"'.addslashes(isset($_SERVER['HTTP_REFERER'])?$_SERVER['HTTP_REFERER']:'').'",'.
|
|
||||||
'"'.addslashes($hLog[1]).'",'.
|
|
||||||
'"'.addslashes($hLog[3]).'",'.
|
|
||||||
'"'.addslashes($_SERVER['HTTP_USER_AGENT']).'",'.
|
|
||||||
'"'.addslashes((isset($_GET['format']))?$_GET['format']:'').'",'.
|
|
||||||
$iNumResults."\n",
|
|
||||||
FILE_APPEND);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,165 +0,0 @@
|
|||||||
amenity airport
|
|
||||||
amenity arts_centre
|
|
||||||
amenity atm
|
|
||||||
amenity auditorium
|
|
||||||
amenity bank
|
|
||||||
amenity bar
|
|
||||||
amenity bench
|
|
||||||
amenity bicycle_parking
|
|
||||||
amenity bicycle_rental
|
|
||||||
amenity brothel
|
|
||||||
amenity bureau_de_change
|
|
||||||
amenity bus_station
|
|
||||||
amenity cafe
|
|
||||||
amenity car_rental
|
|
||||||
amenity car_wash
|
|
||||||
amenity casino
|
|
||||||
amenity cinema
|
|
||||||
amenity clinic
|
|
||||||
amenity club
|
|
||||||
amenity college
|
|
||||||
amenity community_centre
|
|
||||||
amenity courthouse
|
|
||||||
amenity crematorium
|
|
||||||
amenity dentist
|
|
||||||
amenity doctors
|
|
||||||
amenity dormitory
|
|
||||||
amenity drinking_water
|
|
||||||
amenity driving_school
|
|
||||||
amenity embassy
|
|
||||||
amenity emergency_phone
|
|
||||||
amenity fast_food
|
|
||||||
amenity ferry_terminal
|
|
||||||
amenity fire_hydrant
|
|
||||||
amenity fire_station
|
|
||||||
amenity fountain
|
|
||||||
amenity fuel
|
|
||||||
amenity grave_yard
|
|
||||||
amenity hall
|
|
||||||
amenity health_centre
|
|
||||||
amenity hospital
|
|
||||||
amenity hotel
|
|
||||||
amenity hunting_stand
|
|
||||||
amenity ice_cream
|
|
||||||
amenity kindergarten
|
|
||||||
amenity library
|
|
||||||
amenity market
|
|
||||||
amenity marketplace
|
|
||||||
amenity nightclub
|
|
||||||
amenity nursery
|
|
||||||
amenity nursing_home
|
|
||||||
amenity office
|
|
||||||
amenity park
|
|
||||||
amenity parking
|
|
||||||
amenity pharmacy
|
|
||||||
amenity place_of_worship
|
|
||||||
amenity police
|
|
||||||
amenity post_box
|
|
||||||
amenity post_office
|
|
||||||
amenity preschool
|
|
||||||
amenity prison
|
|
||||||
amenity pub
|
|
||||||
amenity public_building
|
|
||||||
amenity public_market
|
|
||||||
amenity reception_area
|
|
||||||
amenity restaurant
|
|
||||||
amenity retirement_home
|
|
||||||
amenity sauna
|
|
||||||
amenity school
|
|
||||||
amenity shelter
|
|
||||||
amenity shop
|
|
||||||
amenity shopping
|
|
||||||
amenity social_club
|
|
||||||
amenity studio
|
|
||||||
amenity supermarket
|
|
||||||
amenity taxi
|
|
||||||
amenity telephone
|
|
||||||
amenity theatre
|
|
||||||
amenity toilets
|
|
||||||
amenity townhall
|
|
||||||
amenity university
|
|
||||||
amenity veterinary
|
|
||||||
amenity waste_basket
|
|
||||||
amenity wifi
|
|
||||||
amenity youth_centre
|
|
||||||
boundary administrative
|
|
||||||
building apartments
|
|
||||||
building block
|
|
||||||
building bunker
|
|
||||||
building chapel
|
|
||||||
building church
|
|
||||||
building commercial
|
|
||||||
building dormitory
|
|
||||||
building entrance
|
|
||||||
building faculty
|
|
||||||
building farm
|
|
||||||
building flats
|
|
||||||
building garage
|
|
||||||
building hospital
|
|
||||||
building hotel
|
|
||||||
building house
|
|
||||||
building industrial
|
|
||||||
building office
|
|
||||||
building public
|
|
||||||
building residential
|
|
||||||
building retail
|
|
||||||
building school
|
|
||||||
building shop
|
|
||||||
building stadium
|
|
||||||
building store
|
|
||||||
building terrace
|
|
||||||
building tower
|
|
||||||
building train_station
|
|
||||||
building university
|
|
||||||
highway bridleway
|
|
||||||
highway bus_stop
|
|
||||||
highway construction
|
|
||||||
highway cycleway
|
|
||||||
highway distance_marker
|
|
||||||
highway emergency_access_point
|
|
||||||
highway footway
|
|
||||||
highway gate
|
|
||||||
highway motorway_junction
|
|
||||||
highway path
|
|
||||||
highway pedestrian
|
|
||||||
highway platform
|
|
||||||
highway primary
|
|
||||||
highway primary_link
|
|
||||||
highway raceway
|
|
||||||
highway road
|
|
||||||
highway secondary
|
|
||||||
highway secondary_link
|
|
||||||
highway services
|
|
||||||
highway steps
|
|
||||||
highway tertiary
|
|
||||||
highway track
|
|
||||||
highway trail
|
|
||||||
highway trunk
|
|
||||||
highway trunk_link
|
|
||||||
highway unsurfaced
|
|
||||||
historic archaeological_site
|
|
||||||
historic battlefield
|
|
||||||
historic building
|
|
||||||
historic castle
|
|
||||||
historic church
|
|
||||||
historic house
|
|
||||||
historic icon
|
|
||||||
historic manor
|
|
||||||
historic memorial
|
|
||||||
historic mine
|
|
||||||
historic monument
|
|
||||||
historic museum
|
|
||||||
historic ruins
|
|
||||||
historic tower
|
|
||||||
historic wayside_cross
|
|
||||||
historic wayside_shrine
|
|
||||||
historic wreck
|
|
||||||
landuse cemetery
|
|
||||||
landuse commercial
|
|
||||||
landuse construction
|
|
||||||
landuse farm
|
|
||||||
landuse farmland
|
|
||||||
landuse farmyard
|
|
||||||
landuse forest
|
|
||||||
landuse grass
|
|
||||||
landuse industrial
|
|
||||||
@@ -11,6 +11,18 @@
|
|||||||
@define('CONST_Database_Web_User', 'www-data');
|
@define('CONST_Database_Web_User', 'www-data');
|
||||||
@define('CONST_Max_Word_Frequency', '50000');
|
@define('CONST_Max_Word_Frequency', '50000');
|
||||||
@define('CONST_Limit_Reindexing', true);
|
@define('CONST_Limit_Reindexing', true);
|
||||||
|
// Set to false to avoid importing extra postcodes for the US.
|
||||||
|
@define('CONST_Use_Extra_US_Postcodes', true);
|
||||||
|
// Set to true after importing Tiger house number data for the US.
|
||||||
|
// Note: The tables must already exist or queries will throw errors.
|
||||||
|
// After changing this setting run ./utils/setup --create-functions
|
||||||
|
// again.
|
||||||
|
@define('CONST_Use_US_Tiger_Data', false);
|
||||||
|
// Set to true after importing other external house number data.
|
||||||
|
// Note: the aux tables must already exist or queries will throw errors.
|
||||||
|
// After changing this setting run ./utils/setup --create-functions
|
||||||
|
// again.
|
||||||
|
@define('CONST_Use_Aux_Location_data', false);
|
||||||
|
|
||||||
// Proxy settings
|
// Proxy settings
|
||||||
@define('CONST_HTTP_Proxy', false);
|
@define('CONST_HTTP_Proxy', false);
|
||||||
@@ -19,13 +31,7 @@
|
|||||||
@define('CONST_HTTP_Proxy_Login', '');
|
@define('CONST_HTTP_Proxy_Login', '');
|
||||||
@define('CONST_HTTP_Proxy_Password', '');
|
@define('CONST_HTTP_Proxy_Password', '');
|
||||||
|
|
||||||
// Software versions
|
|
||||||
@define('CONST_Postgresql_Version', '9.3'); // values: 9.0, ... , 9.4
|
|
||||||
@define('CONST_Postgis_Version', '2.1'); // values: 1.5, 2.0, 2.1
|
|
||||||
|
|
||||||
// Paths
|
// Paths
|
||||||
@define('CONST_Path_Postgresql_Contrib', '/usr/share/postgresql/'.CONST_Postgresql_Version.'/contrib');
|
|
||||||
@define('CONST_Path_Postgresql_Postgis', CONST_Path_Postgresql_Contrib.'/postgis-'.CONST_Postgis_Version);
|
|
||||||
@define('CONST_Osm2pgsql_Binary', CONST_InstallPath.'/osm2pgsql/osm2pgsql');
|
@define('CONST_Osm2pgsql_Binary', CONST_InstallPath.'/osm2pgsql/osm2pgsql');
|
||||||
@define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
|
@define('CONST_Osmosis_Binary', '/usr/bin/osmosis');
|
||||||
@define('CONST_Tiger_Data_Path', CONST_BasePath.'/data/tiger');
|
@define('CONST_Tiger_Data_Path', CONST_BasePath.'/data/tiger');
|
||||||
@@ -81,23 +87,22 @@
|
|||||||
|
|
||||||
// Website settings
|
// Website settings
|
||||||
@define('CONST_NoAccessControl', true);
|
@define('CONST_NoAccessControl', true);
|
||||||
@define('CONST_ClosedForIndexing', false);
|
|
||||||
@define('CONST_ClosedForIndexingExceptionIPs', '');
|
|
||||||
@define('CONST_BlockedIPs', '');
|
@define('CONST_BlockedIPs', '');
|
||||||
@define('CONST_BulkUserIPs', '');
|
@define('CONST_BulkUserIPs', '');
|
||||||
@define('CONST_BlockMessage', ''); // additional info to show for blocked IPs
|
@define('CONST_BlockMessage', ''); // additional info to show for blocked IPs
|
||||||
|
|
||||||
@define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/');
|
@define('CONST_Website_BaseURL', 'http://'.php_uname('n').'/');
|
||||||
@define('CONST_Tile_Default', 'Mapnik');
|
// Language to assume when none is supplied with the query.
|
||||||
|
// When set to false, the local language (i.e. the name tag without suffix)
|
||||||
|
// will be used.
|
||||||
@define('CONST_Default_Language', false);
|
@define('CONST_Default_Language', false);
|
||||||
|
// Appearance of the map in the debug interface.
|
||||||
@define('CONST_Default_Lat', 20.0);
|
@define('CONST_Default_Lat', 20.0);
|
||||||
@define('CONST_Default_Lon', 0.0);
|
@define('CONST_Default_Lon', 0.0);
|
||||||
@define('CONST_Default_Zoom', 2);
|
@define('CONST_Default_Zoom', 2);
|
||||||
@define('CONST_Map_Tile_URL', 'http://{s}.tile.osm.org/{z}/{x}/{y}.png');
|
@define('CONST_Map_Tile_URL', 'http://{s}.tile.osm.org/{z}/{x}/{y}.png');
|
||||||
@define('CONST_Map_Tile_Attribution', ''); // Set if tile source isn't osm.org
|
@define('CONST_Map_Tile_Attribution', ''); // Set if tile source isn't osm.org
|
||||||
|
|
||||||
@define('CONST_Search_AreaPolygons_Enabled', true);
|
|
||||||
@define('CONST_Search_AreaPolygons', true);
|
@define('CONST_Search_AreaPolygons', true);
|
||||||
|
|
||||||
@define('CONST_Search_BatchMode', false);
|
@define('CONST_Search_BatchMode', false);
|
||||||
@@ -108,16 +113,19 @@
|
|||||||
// When set to false only selected languages alloow reverse search.
|
// When set to false only selected languages alloow reverse search.
|
||||||
@define('CONST_Search_ReversePlanForAll', true);
|
@define('CONST_Search_ReversePlanForAll', true);
|
||||||
|
|
||||||
|
// Maximum number of OSM ids that may be queried at once
|
||||||
|
// for the places endpoint.
|
||||||
@define('CONST_Places_Max_ID_count', 50);
|
@define('CONST_Places_Max_ID_count', 50);
|
||||||
|
|
||||||
// Set to zero to disable polygon output
|
// Number of different geometry formats that may be queried in parallel.
|
||||||
|
// Set to zero to disable polygon output.
|
||||||
@define('CONST_PolygonOutput_MaximumTypes', 1);
|
@define('CONST_PolygonOutput_MaximumTypes', 1);
|
||||||
|
|
||||||
// Log settings
|
// Log settings
|
||||||
@define('CONST_Log_DB', true);
|
// Set to true to log into new_query_log table.
|
||||||
|
// You should set up a cron job that regularly clears out this table.
|
||||||
|
@define('CONST_Log_DB', false);
|
||||||
|
// Set to a file name to enable logging to a file.
|
||||||
@define('CONST_Log_File', false);
|
@define('CONST_Log_File', false);
|
||||||
@define('CONST_Log_File_Format', 'TODO'); // Currently hard coded
|
|
||||||
@define('CONST_Log_File_SearchLog', '');
|
|
||||||
@define('CONST_Log_File_ReverseLog', '');
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
6
sql/aux_tables.sql
Normal file
6
sql/aux_tables.sql
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
CREATE TABLE location_property_aux () INHERITS (location_property);
|
||||||
|
CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
|
||||||
|
CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
|
||||||
|
CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
|
||||||
|
GRANT SELECT ON location_property_aux TO "{www-user}";
|
||||||
|
|
||||||
@@ -1,3 +1,15 @@
|
|||||||
|
-- Splits the line at the given point and returns the two parts
|
||||||
|
-- in a multilinestring.
|
||||||
|
CREATE OR REPLACE FUNCTION split_line_on_node(line GEOMETRY, point GEOMETRY)
|
||||||
|
RETURNS GEOMETRY
|
||||||
|
AS $$
|
||||||
|
BEGIN
|
||||||
|
RETURN ST_Split(ST_Snap(line, point, 0.0005), point);
|
||||||
|
END;
|
||||||
|
$$
|
||||||
|
LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
|
||||||
CREATE OR REPLACE FUNCTION geometry_sector(partition INTEGER, place geometry) RETURNS INTEGER
|
CREATE OR REPLACE FUNCTION geometry_sector(partition INTEGER, place geometry) RETURNS INTEGER
|
||||||
AS $$
|
AS $$
|
||||||
DECLARE
|
DECLARE
|
||||||
@@ -990,7 +1002,7 @@ BEGIN
|
|||||||
|
|
||||||
--DEBUG: RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
|
--DEBUG: RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
|
||||||
|
|
||||||
RETURN NEW; -- @DIFFUPDATES@ The following is not needed until doing diff updates, and slows the main index process down
|
RETURN NEW; -- %DIFFUPDATES% The following is not needed until doing diff updates, and slows the main index process down
|
||||||
|
|
||||||
IF NEW.rank_address > 0 THEN
|
IF NEW.rank_address > 0 THEN
|
||||||
IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
|
IF (ST_GeometryType(NEW.geometry) in ('ST_Polygon','ST_MultiPolygon') AND ST_IsValid(NEW.geometry)) THEN
|
||||||
@@ -1606,6 +1618,7 @@ BEGIN
|
|||||||
NEW.parent_place_id = 0;
|
NEW.parent_place_id = 0;
|
||||||
parent_place_id_rank = 0;
|
parent_place_id_rank = 0;
|
||||||
|
|
||||||
|
|
||||||
-- convert isin to array of tokenids
|
-- convert isin to array of tokenids
|
||||||
isin_tokens := '{}'::int[];
|
isin_tokens := '{}'::int[];
|
||||||
IF NEW.isin IS NOT NULL THEN
|
IF NEW.isin IS NOT NULL THEN
|
||||||
@@ -1645,6 +1658,7 @@ BEGIN
|
|||||||
END IF;
|
END IF;
|
||||||
END IF;
|
END IF;
|
||||||
|
|
||||||
|
-- %NOTIGERDATA% IF 0 THEN
|
||||||
-- for the USA we have an additional address table. Merge in zip codes from there too
|
-- for the USA we have an additional address table. Merge in zip codes from there too
|
||||||
IF NEW.rank_search = 26 AND NEW.calculated_country_code = 'us' THEN
|
IF NEW.rank_search = 26 AND NEW.calculated_country_code = 'us' THEN
|
||||||
FOR location IN SELECT distinct postcode from location_property_tiger where parent_place_id = NEW.place_id LOOP
|
FOR location IN SELECT distinct postcode from location_property_tiger where parent_place_id = NEW.place_id LOOP
|
||||||
@@ -1657,6 +1671,7 @@ BEGIN
|
|||||||
nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
|
nameaddress_vector := array_merge(nameaddress_vector, ARRAY[address_street_word_id]);
|
||||||
END LOOP;
|
END LOOP;
|
||||||
END IF;
|
END IF;
|
||||||
|
-- %NOTIGERDATA% END IF;
|
||||||
|
|
||||||
-- RAISE WARNING 'ISIN: %', isin_tokens;
|
-- RAISE WARNING 'ISIN: %', isin_tokens;
|
||||||
|
|
||||||
@@ -2315,7 +2330,6 @@ DECLARE
|
|||||||
countryname HSTORE;
|
countryname HSTORE;
|
||||||
hadcountry BOOLEAN;
|
hadcountry BOOLEAN;
|
||||||
BEGIN
|
BEGIN
|
||||||
|
|
||||||
-- first query osmline (interpolation lines)
|
-- first query osmline (interpolation lines)
|
||||||
select parent_place_id, calculated_country_code, 30, postcode, null, 'place', 'house' from location_property_osmline
|
select parent_place_id, calculated_country_code, 30, postcode, null, 'place', 'house' from location_property_osmline
|
||||||
WHERE place_id = in_place_id AND in_housenumber>=startnumber AND in_housenumber <= endnumber
|
WHERE place_id = in_place_id AND in_housenumber>=startnumber AND in_housenumber <= endnumber
|
||||||
@@ -2323,8 +2337,9 @@ BEGIN
|
|||||||
IF for_place_id IS NOT NULL THEN
|
IF for_place_id IS NOT NULL THEN
|
||||||
searchhousenumber = in_housenumber::text;
|
searchhousenumber = in_housenumber::text;
|
||||||
END IF;
|
END IF;
|
||||||
|
|
||||||
--then query tiger data
|
--then query tiger data
|
||||||
|
-- %NOTIGERDATA% IF 0 THEN
|
||||||
IF for_place_id IS NULL THEN
|
IF for_place_id IS NULL THEN
|
||||||
select parent_place_id,'us', 30, postcode, null, 'place', 'house' from location_property_tiger
|
select parent_place_id,'us', 30, postcode, null, 'place', 'house' from location_property_tiger
|
||||||
WHERE place_id = in_place_id AND in_housenumber>=startnumber AND in_housenumber <= endnumber
|
WHERE place_id = in_place_id AND in_housenumber>=startnumber AND in_housenumber <= endnumber
|
||||||
@@ -2333,13 +2348,16 @@ BEGIN
|
|||||||
searchhousenumber = in_housenumber::text;
|
searchhousenumber = in_housenumber::text;
|
||||||
END IF;
|
END IF;
|
||||||
END IF;
|
END IF;
|
||||||
|
-- %NOTIGERDATA% END IF;
|
||||||
|
|
||||||
|
-- %NOAUXDATA% IF 0 THEN
|
||||||
IF for_place_id IS NULL THEN
|
IF for_place_id IS NULL THEN
|
||||||
select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux
|
select parent_place_id,'us', housenumber, 30, postcode, null, 'place', 'house' from location_property_aux
|
||||||
WHERE place_id = in_place_id
|
WHERE place_id = in_place_id
|
||||||
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
|
INTO for_place_id,searchcountrycode, searchhousenumber, searchrankaddress, searchpostcode, searchhousename, searchclass, searchtype;
|
||||||
END IF;
|
END IF;
|
||||||
|
-- %NOAUXDATA% END IF;
|
||||||
|
|
||||||
IF for_place_id IS NULL THEN
|
IF for_place_id IS NULL THEN
|
||||||
select parent_place_id, calculated_country_code, housenumber, rank_search, postcode, name, class, type from placex
|
select parent_place_id, calculated_country_code, housenumber, rank_search, postcode, name, class, type from placex
|
||||||
WHERE place_id = in_place_id and rank_address = 30
|
WHERE place_id = in_place_id and rank_address = 30
|
||||||
|
|||||||
@@ -19,10 +19,6 @@ CREATE INDEX idx_location_area_country_place_id ON location_area_country USING B
|
|||||||
|
|
||||||
CREATE INDEX idx_search_name_country_centroid ON search_name_country USING GIST (centroid) {ts:address-index};
|
CREATE INDEX idx_search_name_country_centroid ON search_name_country USING GIST (centroid) {ts:address-index};
|
||||||
|
|
||||||
-- start
|
|
||||||
CREATE INDEX idx_location_property_-partition-_centroid ON location_property_-partition- USING GIST (centroid) {ts:address-index};
|
|
||||||
-- end
|
|
||||||
|
|
||||||
DROP INDEX IF EXISTS place_id_idx;
|
DROP INDEX IF EXISTS place_id_idx;
|
||||||
CREATE UNIQUE INDEX idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index};
|
CREATE UNIQUE INDEX idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index};
|
||||||
|
|
||||||
|
|||||||
@@ -51,11 +51,6 @@ CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USI
|
|||||||
CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid) {ts:address-index};
|
CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid) {ts:address-index};
|
||||||
CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off) {ts:address-index};
|
CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off) {ts:address-index};
|
||||||
|
|
||||||
CREATE TABLE location_property_-partition- () INHERITS (location_property) {ts:aux-data};
|
|
||||||
CREATE INDEX idx_location_property_-partition-_place_id ON location_property_-partition- USING BTREE (place_id) {ts:aux-index};
|
|
||||||
CREATE INDEX idx_location_property_-partition-_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id) {ts:aux-index};
|
|
||||||
CREATE INDEX idx_location_property_-partition-_housenumber_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id, housenumber) {ts:aux-index};
|
|
||||||
|
|
||||||
CREATE TABLE location_road_-partition- (
|
CREATE TABLE location_road_-partition- (
|
||||||
partition integer,
|
partition integer,
|
||||||
place_id BIGINT,
|
place_id BIGINT,
|
||||||
|
|||||||
@@ -1,14 +0,0 @@
|
|||||||
-- Splits the line at the given point and returns the two parts
|
|
||||||
-- in a multilinestring.
|
|
||||||
CREATE OR REPLACE FUNCTION split_line_on_node(line GEOMETRY, point GEOMETRY)
|
|
||||||
RETURNS GEOMETRY
|
|
||||||
AS $$
|
|
||||||
DECLARE
|
|
||||||
frac FLOAT;
|
|
||||||
BEGIN
|
|
||||||
frac := ST_Line_Locate_Point(line, point);
|
|
||||||
RETURN ST_Collect(ST_Line_Substring(line, 0, frac),
|
|
||||||
ST_Line_Substring(line, frac, 1));
|
|
||||||
END
|
|
||||||
$$
|
|
||||||
LANGUAGE plpgsql;
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
-- Splits the line at the given point and returns the two parts
|
|
||||||
-- in a multilinestring.
|
|
||||||
CREATE OR REPLACE FUNCTION split_line_on_node(line GEOMETRY, point GEOMETRY)
|
|
||||||
RETURNS GEOMETRY
|
|
||||||
AS $$
|
|
||||||
BEGIN
|
|
||||||
RETURN ST_Split(ST_Snap(line, point, 0.0005), point);
|
|
||||||
END;
|
|
||||||
$$
|
|
||||||
LANGUAGE plpgsql;
|
|
||||||
@@ -1,155 +0,0 @@
|
|||||||
drop table import_npi_log;
|
|
||||||
CREATE TABLE import_npi_log (
|
|
||||||
npiid integer,
|
|
||||||
batchend timestamp,
|
|
||||||
batchsize integer,
|
|
||||||
starttime timestamp,
|
|
||||||
endtime timestamp,
|
|
||||||
event text
|
|
||||||
);
|
|
||||||
|
|
||||||
drop table IF EXISTS word;
|
|
||||||
CREATE TABLE word (
|
|
||||||
word_id INTEGER,
|
|
||||||
word_token text,
|
|
||||||
word_trigram text,
|
|
||||||
word text,
|
|
||||||
class text,
|
|
||||||
type text,
|
|
||||||
country_code varchar(2),
|
|
||||||
search_name_count INTEGER,
|
|
||||||
operator TEXT
|
|
||||||
);
|
|
||||||
SELECT AddGeometryColumn('word', 'location', 4326, 'GEOMETRY', 2);
|
|
||||||
CREATE INDEX idx_word_word_id on word USING BTREE (word_id);
|
|
||||||
CREATE INDEX idx_word_word_token on word USING BTREE (word_token);
|
|
||||||
GRANT SELECT ON word TO "www-data" ;
|
|
||||||
DROP SEQUENCE seq_word;
|
|
||||||
CREATE SEQUENCE seq_word start 1;
|
|
||||||
|
|
||||||
drop table IF EXISTS location_property CASCADE;
|
|
||||||
CREATE TABLE location_property (
|
|
||||||
place_id BIGINT,
|
|
||||||
partition integer,
|
|
||||||
parent_place_id BIGINT,
|
|
||||||
housenumber TEXT,
|
|
||||||
postcode TEXT
|
|
||||||
);
|
|
||||||
SELECT AddGeometryColumn('location_property', 'centroid', 4326, 'POINT', 2);
|
|
||||||
|
|
||||||
CREATE TABLE location_property_aux () INHERITS (location_property);
|
|
||||||
CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
|
|
||||||
CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
|
|
||||||
CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
|
|
||||||
|
|
||||||
CREATE TABLE location_property_tiger () INHERITS (location_property);
|
|
||||||
CREATE INDEX idx_location_property_tiger_place_id ON location_property_tiger USING BTREE (place_id);
|
|
||||||
CREATE INDEX idx_location_property_tiger_parent_place_id ON location_property_tiger USING BTREE (parent_place_id);
|
|
||||||
CREATE INDEX idx_location_property_tiger_housenumber_parent_place_id ON location_property_tiger USING BTREE (parent_place_id, housenumber);
|
|
||||||
|
|
||||||
drop table IF EXISTS search_name_blank CASCADE;
|
|
||||||
CREATE TABLE search_name_blank (
|
|
||||||
place_id BIGINT,
|
|
||||||
search_rank integer,
|
|
||||||
address_rank integer,
|
|
||||||
importance FLOAT,
|
|
||||||
country_code varchar(2),
|
|
||||||
name_vector integer[],
|
|
||||||
nameaddress_vector integer[]
|
|
||||||
);
|
|
||||||
SELECT AddGeometryColumn('search_name_blank', 'centroid', 4326, 'GEOMETRY', 2);
|
|
||||||
|
|
||||||
drop table IF EXISTS search_name;
|
|
||||||
CREATE TABLE search_name () INHERITS (search_name_blank);
|
|
||||||
CREATE INDEX search_name_name_vector_idx ON search_name USING GIN (name_vector gin__int_ops) WITH (fastupdate = off);
|
|
||||||
CREATE INDEX searchnameplacesearch_search_nameaddress_vector_idx ON search_name USING GIN (nameaddress_vector gin__int_ops) WITH (fastupdate = off);
|
|
||||||
CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid);
|
|
||||||
CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id);
|
|
||||||
|
|
||||||
drop table IF EXISTS place_addressline;
|
|
||||||
CREATE TABLE place_addressline (
|
|
||||||
place_id BIGINT,
|
|
||||||
address_place_id BIGINT,
|
|
||||||
fromarea boolean,
|
|
||||||
isaddress boolean,
|
|
||||||
distance float,
|
|
||||||
cached_rank_address integer
|
|
||||||
);
|
|
||||||
CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id);
|
|
||||||
CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id);
|
|
||||||
|
|
||||||
drop table IF EXISTS place_boundingbox CASCADE;
|
|
||||||
CREATE TABLE place_boundingbox (
|
|
||||||
place_id BIGINT,
|
|
||||||
minlat float,
|
|
||||||
maxlat float,
|
|
||||||
minlon float,
|
|
||||||
maxlon float,
|
|
||||||
numfeatures integer,
|
|
||||||
area float
|
|
||||||
);
|
|
||||||
CREATE INDEX idx_place_boundingbox_place_id on place_boundingbox USING BTREE (place_id);
|
|
||||||
SELECT AddGeometryColumn('place_boundingbox', 'outline', 4326, 'GEOMETRY', 2);
|
|
||||||
CREATE INDEX idx_place_boundingbox_outline ON place_boundingbox USING GIST (outline);
|
|
||||||
GRANT SELECT on place_boundingbox to "www-data" ;
|
|
||||||
GRANT INSERT on place_boundingbox to "www-data" ;
|
|
||||||
|
|
||||||
drop table country;
|
|
||||||
CREATE TABLE country (
|
|
||||||
country_code varchar(2),
|
|
||||||
country_name hstore,
|
|
||||||
country_default_language_code varchar(2)
|
|
||||||
);
|
|
||||||
SELECT AddGeometryColumn('country', 'geometry', 4326, 'POLYGON', 2);
|
|
||||||
insert into country select iso3166::varchar(2), 'name:en'->cntry_name, null,
|
|
||||||
ST_Transform(geometryn(the_geom, generate_series(1, numgeometries(the_geom))), 4326) from worldboundaries;
|
|
||||||
CREATE INDEX idx_country_country_code ON country USING BTREE (country_code);
|
|
||||||
CREATE INDEX idx_country_geometry ON country USING GIST (geometry);
|
|
||||||
|
|
||||||
drop table placex;
|
|
||||||
CREATE TABLE placex (
|
|
||||||
place_id BIGINT NOT NULL,
|
|
||||||
partition integer,
|
|
||||||
osm_type char(1),
|
|
||||||
osm_id INTEGER,
|
|
||||||
class TEXT NOT NULL,
|
|
||||||
type TEXT NOT NULL,
|
|
||||||
name HSTORE,
|
|
||||||
admin_level INTEGER,
|
|
||||||
housenumber TEXT,
|
|
||||||
street TEXT,
|
|
||||||
isin TEXT,
|
|
||||||
postcode TEXT,
|
|
||||||
country_code varchar(2),
|
|
||||||
extratags HSTORE,
|
|
||||||
parent_place_id BIGINT,
|
|
||||||
linked_place_id BIGINT,
|
|
||||||
rank_address INTEGER,
|
|
||||||
rank_search INTEGER,
|
|
||||||
importance FLOAT,
|
|
||||||
indexed_status INTEGER,
|
|
||||||
indexed_date TIMESTAMP,
|
|
||||||
geometry_sector INTEGER
|
|
||||||
);
|
|
||||||
SELECT AddGeometryColumn('placex', 'geometry', 4326, 'GEOMETRY', 2);
|
|
||||||
CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id);
|
|
||||||
CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id);
|
|
||||||
CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search);
|
|
||||||
CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address);
|
|
||||||
CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry);
|
|
||||||
CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) where parent_place_id IS NOT NULL;
|
|
||||||
|
|
||||||
DROP SEQUENCE seq_place;
|
|
||||||
CREATE SEQUENCE seq_place start 1;
|
|
||||||
GRANT SELECT on placex to "www-data" ;
|
|
||||||
GRANT UPDATE ON placex to "www-data" ;
|
|
||||||
GRANT SELECT ON search_name to "www-data" ;
|
|
||||||
GRANT DELETE on search_name to "www-data" ;
|
|
||||||
GRANT INSERT on search_name to "www-data" ;
|
|
||||||
GRANT SELECT on place_addressline to "www-data" ;
|
|
||||||
GRANT INSERT ON place_addressline to "www-data" ;
|
|
||||||
GRANT DELETE on place_addressline to "www-data" ;
|
|
||||||
GRANT SELECT ON seq_word to "www-data" ;
|
|
||||||
GRANT UPDATE ON seq_word to "www-data" ;
|
|
||||||
GRANT INSERT ON word to "www-data" ;
|
|
||||||
GRANT SELECT on country to "www-data" ;
|
|
||||||
@@ -13,29 +13,6 @@ CREATE TABLE import_osmosis_log (
|
|||||||
event text
|
event text
|
||||||
);
|
);
|
||||||
|
|
||||||
drop table if exists import_npi_log;
|
|
||||||
CREATE TABLE import_npi_log (
|
|
||||||
npiid integer,
|
|
||||||
batchend timestamp,
|
|
||||||
batchsize integer,
|
|
||||||
starttime timestamp,
|
|
||||||
endtime timestamp,
|
|
||||||
event text
|
|
||||||
);
|
|
||||||
|
|
||||||
--drop table IF EXISTS query_log;
|
|
||||||
CREATE TABLE query_log (
|
|
||||||
starttime timestamp,
|
|
||||||
query text,
|
|
||||||
ipaddress text,
|
|
||||||
endtime timestamp,
|
|
||||||
results integer
|
|
||||||
);
|
|
||||||
CREATE INDEX idx_query_log ON query_log USING BTREE (starttime);
|
|
||||||
GRANT SELECT ON query_log TO "{www-user}" ;
|
|
||||||
GRANT INSERT ON query_log TO "{www-user}" ;
|
|
||||||
GRANT UPDATE ON query_log TO "{www-user}" ;
|
|
||||||
|
|
||||||
CREATE TABLE new_query_log (
|
CREATE TABLE new_query_log (
|
||||||
type text,
|
type text,
|
||||||
starttime timestamp,
|
starttime timestamp,
|
||||||
@@ -43,6 +20,7 @@ CREATE TABLE new_query_log (
|
|||||||
useragent text,
|
useragent text,
|
||||||
language text,
|
language text,
|
||||||
query text,
|
query text,
|
||||||
|
searchterm text,
|
||||||
endtime timestamp,
|
endtime timestamp,
|
||||||
results integer,
|
results integer,
|
||||||
format text,
|
format text,
|
||||||
@@ -56,9 +34,6 @@ GRANT SELECT ON new_query_log TO "{www-user}" ;
|
|||||||
GRANT SELECT ON TABLE country_name TO "{www-user}";
|
GRANT SELECT ON TABLE country_name TO "{www-user}";
|
||||||
GRANT SELECT ON TABLE gb_postcode TO "{www-user}";
|
GRANT SELECT ON TABLE gb_postcode TO "{www-user}";
|
||||||
|
|
||||||
create view vw_search_query_log as SELECT substr(query, 1, 50) AS query, starttime, endtime - starttime AS duration, substr(useragent, 1, 20) as
|
|
||||||
useragent, language, results, ipaddress FROM new_query_log WHERE type = 'search' ORDER BY starttime DESC;
|
|
||||||
|
|
||||||
drop table IF EXISTS word;
|
drop table IF EXISTS word;
|
||||||
CREATE TABLE word (
|
CREATE TABLE word (
|
||||||
word_id INTEGER,
|
word_id INTEGER,
|
||||||
@@ -129,7 +104,6 @@ CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline (place_id)
|
|||||||
CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline (parent_place_id) {ts:search-index};
|
CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline (parent_place_id) {ts:search-index};
|
||||||
GRANT SELECT ON location_property_osmline TO "{www-user}";
|
GRANT SELECT ON location_property_osmline TO "{www-user}";
|
||||||
|
|
||||||
|
|
||||||
drop table IF EXISTS search_name;
|
drop table IF EXISTS search_name;
|
||||||
CREATE TABLE search_name (
|
CREATE TABLE search_name (
|
||||||
place_id BIGINT,
|
place_id BIGINT,
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
\a
|
|
||||||
\t
|
|
||||||
\o /tmp/bigintupdate.sql
|
|
||||||
select 'alter table "'||relname||'" alter '||attname||' type bigint;' from pg_attribute join pg_class on
|
|
||||||
(attrelid = oid) where attname like '%place_id%' and attnum > 0 and relkind = 'r'::"char" and atttypid = 23
|
|
||||||
and not relname::text ~ '^.*_[0-9]+$' order by 'alter table "'||relname||'" alter '||attname||' type
|
|
||||||
bigint;';
|
|
||||||
\o
|
|
||||||
\i /tmp/bigintupdate.sql
|
|
||||||
@@ -175,8 +175,10 @@ def db_template_setup():
|
|||||||
conn = psycopg2.connect(database=world.config.template_db)
|
conn = psycopg2.connect(database=world.config.template_db)
|
||||||
psycopg2.extras.register_hstore(conn, globally=False, unicode=True)
|
psycopg2.extras.register_hstore(conn, globally=False, unicode=True)
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
for table in ('gb_postcode', 'us_postcode', 'us_state', 'us_statecounty'):
|
for table in ('gb_postcode', 'us_postcode'):
|
||||||
cur.execute('TRUNCATE TABLE %s' % (table,))
|
cur.execute("select * from pg_tables where tablename = '%s'" % (table, ))
|
||||||
|
if cur.rowcount > 0:
|
||||||
|
cur.execute('TRUNCATE TABLE %s' % (table,))
|
||||||
conn.commit()
|
conn.commit()
|
||||||
conn.close()
|
conn.close()
|
||||||
# execute osm2pgsql on an empty file to get the right tables
|
# execute osm2pgsql on an empty file to get the right tables
|
||||||
|
|||||||
172
utils/setup.php
172
utils/setup.php
@@ -24,7 +24,6 @@
|
|||||||
array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
|
array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
|
||||||
array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
|
array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
|
||||||
array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
|
array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
|
||||||
array('create-minimal-tables', '', 0, 1, 0, 0, 'bool', 'Create minimal main tables'),
|
|
||||||
array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
|
array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
|
||||||
array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
|
array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
|
||||||
array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
|
array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
|
||||||
@@ -37,7 +36,6 @@
|
|||||||
array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
|
array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
|
||||||
array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
|
array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
|
||||||
array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
|
array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
|
||||||
array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
|
|
||||||
array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
|
array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
|
||||||
array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
|
array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
|
||||||
array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'),
|
array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'),
|
||||||
@@ -92,8 +90,6 @@
|
|||||||
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
|
$aDSNInfo = DB::parseDSN(CONST_Database_DSN);
|
||||||
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
|
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
|
||||||
|
|
||||||
$fPostgisVersion = (float) CONST_Postgis_Version;
|
|
||||||
|
|
||||||
if ($aCMDResult['create-db'] || $aCMDResult['all'])
|
if ($aCMDResult['create-db'] || $aCMDResult['all'])
|
||||||
{
|
{
|
||||||
echo "Create DB\n";
|
echo "Create DB\n";
|
||||||
@@ -114,40 +110,39 @@
|
|||||||
|
|
||||||
$oDB =& getDB();
|
$oDB =& getDB();
|
||||||
|
|
||||||
$sVersionString = $oDB->getOne('select version()');
|
$fPostgresVersion = getPostgresVersion($oDB);
|
||||||
preg_match('#PostgreSQL ([0-9]+)[.]([0-9]+)[^0-9]#', $sVersionString, $aMatches);
|
echo 'Postgres version found: '.$fPostgresVersion."\n";
|
||||||
if (CONST_Postgresql_Version != $aMatches[1].'.'.$aMatches[2])
|
|
||||||
|
if ($fPostgresVersion < 9.1)
|
||||||
{
|
{
|
||||||
echo "ERROR: PostgreSQL version is not correct. Expected ".CONST_Postgresql_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
|
fail("Minimum supported version of Postgresql is 9.1.");
|
||||||
exit;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
passthru('createlang plpgsql -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']);
|
pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
|
||||||
$pgver = (float) CONST_Postgresql_Version;
|
pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
|
||||||
if ($pgver < 9.1) {
|
|
||||||
pgsqlRunScriptFile(CONST_Path_Postgresql_Contrib.'/hstore.sql');
|
// For extratags and namedetails the hstore_to_json converter is
|
||||||
pgsqlRunScriptFile(CONST_BasePath.'/sql/hstore_compatability_9_0.sql');
|
// needed which is only available from Postgresql 9.3+. For older
|
||||||
} else {
|
// versions add a dummy function that returns nothing.
|
||||||
pgsqlRunScript('CREATE EXTENSION hstore');
|
$iNumFunc = $oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'");
|
||||||
|
if (PEAR::isError($iNumFunc))
|
||||||
|
{
|
||||||
|
fail("Cannot query stored procedures.", $iNumFunc);
|
||||||
|
}
|
||||||
|
if ($iNumFunc == 0)
|
||||||
|
{
|
||||||
|
pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable");
|
||||||
|
echo "WARNING: Postgresql is too old. extratags and namedetails API not available.";
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($fPostgisVersion < 2.0) {
|
$fPostgisVersion = getPostgisVersion($oDB);
|
||||||
pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/postgis.sql');
|
echo 'Postgis version found: '.$fPostgisVersion."\n";
|
||||||
pgsqlRunScriptFile(CONST_Path_Postgresql_Postgis.'/spatial_ref_sys.sql');
|
|
||||||
} else {
|
if ($fPostgisVersion < 2.1)
|
||||||
pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
|
{
|
||||||
}
|
|
||||||
if ($fPostgisVersion < 2.1) {
|
|
||||||
// Function was renamed in 2.1 and throws an annoying deprecation warning
|
// Function was renamed in 2.1 and throws an annoying deprecation warning
|
||||||
pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
|
pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint');
|
||||||
}
|
}
|
||||||
$sVersionString = $oDB->getOne('select postgis_full_version()');
|
|
||||||
preg_match('#POSTGIS="([0-9]+)[.]([0-9]+)[.]([0-9]+)( r([0-9]+))?"#', $sVersionString, $aMatches);
|
|
||||||
if (CONST_Postgis_Version != $aMatches[1].'.'.$aMatches[2])
|
|
||||||
{
|
|
||||||
echo "ERROR: PostGIS version is not correct. Expected ".CONST_Postgis_Version." found ".$aMatches[1].'.'.$aMatches[2]."\n";
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
|
pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
|
||||||
pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
|
pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
|
||||||
@@ -161,9 +156,10 @@
|
|||||||
{
|
{
|
||||||
echo "WARNING: external UK postcode table not found.\n";
|
echo "WARNING: external UK postcode table not found.\n";
|
||||||
}
|
}
|
||||||
pgsqlRunScriptFile(CONST_BasePath.'/data/us_statecounty.sql');
|
if (CONST_Use_Extra_US_Postcodes)
|
||||||
pgsqlRunScriptFile(CONST_BasePath.'/data/us_state.sql');
|
{
|
||||||
pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
|
pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
|
||||||
|
}
|
||||||
|
|
||||||
if ($aCMDResult['no-partitions'])
|
if ($aCMDResult['no-partitions'])
|
||||||
{
|
{
|
||||||
@@ -221,51 +217,7 @@
|
|||||||
echo "Functions\n";
|
echo "Functions\n";
|
||||||
$bDidSomething = true;
|
$bDidSomething = true;
|
||||||
if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built");
|
if (!file_exists(CONST_InstallPath.'/module/nominatim.so')) fail("nominatim module not built");
|
||||||
$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
|
create_sql_functions($aCMDResult);
|
||||||
$sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate);
|
|
||||||
if ($aCMDResult['enable-diff-updates']) $sTemplate = str_replace('RETURN NEW; -- @DIFFUPDATES@', '--', $sTemplate);
|
|
||||||
if ($aCMDResult['enable-debug-statements']) $sTemplate = str_replace('--DEBUG:', '', $sTemplate);
|
|
||||||
if (CONST_Limit_Reindexing) $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
|
|
||||||
pgsqlRunScript($sTemplate);
|
|
||||||
|
|
||||||
if ($fPostgisVersion < 2.0) {
|
|
||||||
echo "Helper functions for postgis < 2.0\n";
|
|
||||||
$sTemplate = file_get_contents(CONST_BasePath.'/sql/postgis_15_aux.sql');
|
|
||||||
} else {
|
|
||||||
echo "Helper functions for postgis >= 2.0\n";
|
|
||||||
$sTemplate = file_get_contents(CONST_BasePath.'/sql/postgis_20_aux.sql');
|
|
||||||
}
|
|
||||||
pgsqlRunScript($sTemplate);
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($aCMDResult['create-minimal-tables'])
|
|
||||||
{
|
|
||||||
echo "Minimal Tables\n";
|
|
||||||
$bDidSomething = true;
|
|
||||||
pgsqlRunScriptFile(CONST_BasePath.'/sql/tables-minimal.sql');
|
|
||||||
|
|
||||||
$sScript = '';
|
|
||||||
|
|
||||||
// Backstop the import process - easliest possible import id
|
|
||||||
$sScript .= "insert into import_npi_log values (18022);\n";
|
|
||||||
|
|
||||||
$hFile = @fopen(CONST_BasePath.'/settings/partitionedtags.def', "r");
|
|
||||||
if (!$hFile) fail('unable to open list of partitions: '.CONST_BasePath.'/settings/partitionedtags.def');
|
|
||||||
|
|
||||||
while (($sLine = fgets($hFile, 4096)) !== false && $sLine && substr($sLine,0,1) !='#')
|
|
||||||
{
|
|
||||||
list($sClass, $sType) = explode(' ', trim($sLine));
|
|
||||||
$sScript .= "create table place_classtype_".$sClass."_".$sType." as ";
|
|
||||||
$sScript .= "select place_id as place_id,geometry as centroid from placex limit 0;\n";
|
|
||||||
|
|
||||||
$sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_centroid ";
|
|
||||||
$sScript .= "ON place_classtype_".$sClass."_".$sType." USING GIST (centroid);\n";
|
|
||||||
|
|
||||||
$sScript .= "CREATE INDEX idx_place_classtype_".$sClass."_".$sType."_place_id ";
|
|
||||||
$sScript .= "ON place_classtype_".$sClass."_".$sType." USING btree(place_id);\n";
|
|
||||||
}
|
|
||||||
fclose($hFile);
|
|
||||||
pgsqlRunScript($sScript);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($aCMDResult['create-tables'] || $aCMDResult['all'])
|
if ($aCMDResult['create-tables'] || $aCMDResult['all'])
|
||||||
@@ -291,10 +243,7 @@
|
|||||||
|
|
||||||
// re-run the functions
|
// re-run the functions
|
||||||
echo "Functions\n";
|
echo "Functions\n";
|
||||||
$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
|
create_sql_functions($aCMDResult);
|
||||||
$sTemplate = str_replace('{modulepath}',
|
|
||||||
CONST_InstallPath.'/module', $sTemplate);
|
|
||||||
pgsqlRunScript($sTemplate);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
|
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all'])
|
||||||
@@ -597,10 +546,14 @@
|
|||||||
$sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
|
$sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
|
||||||
$sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
|
$sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
|
||||||
if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
|
if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
|
||||||
$sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
|
|
||||||
$sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
|
if (CONST_Use_Extra_US_Postcodes)
|
||||||
$sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
|
{
|
||||||
if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
|
$sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
|
||||||
|
$sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
|
||||||
|
$sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";
|
||||||
|
if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($aCMDResult['osmosis-init'] || ($aCMDResult['all'] && !$aCMDResult['drop'])) // no use doing osmosis-init when dropping update tables
|
if ($aCMDResult['osmosis-init'] || ($aCMDResult['all'] && !$aCMDResult['drop'])) // no use doing osmosis-init when dropping update tables
|
||||||
@@ -701,7 +654,6 @@
|
|||||||
{
|
{
|
||||||
$bDidSomething = true;
|
$bDidSomething = true;
|
||||||
$sOutputFile = '';
|
$sOutputFile = '';
|
||||||
if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
|
|
||||||
$sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
|
$sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile;
|
||||||
passthruCheckReturn($sBaseCmd.' -R 4');
|
passthruCheckReturn($sBaseCmd.' -R 4');
|
||||||
if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
|
if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE');
|
||||||
@@ -714,14 +666,6 @@
|
|||||||
{
|
{
|
||||||
echo "Search indices\n";
|
echo "Search indices\n";
|
||||||
$bDidSomething = true;
|
$bDidSomething = true;
|
||||||
$oDB =& getDB();
|
|
||||||
$sSQL = 'select distinct partition from country_name';
|
|
||||||
$aPartitions = $oDB->getCol($sSQL);
|
|
||||||
if (PEAR::isError($aPartitions))
|
|
||||||
{
|
|
||||||
fail($aPartitions->getMessage());
|
|
||||||
}
|
|
||||||
if (!$aCMDResult['no-partitions']) $aPartitions[] = 0;
|
|
||||||
|
|
||||||
$sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
|
$sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
|
||||||
$sTemplate = replace_tablespace('{ts:address-index}',
|
$sTemplate = replace_tablespace('{ts:address-index}',
|
||||||
@@ -730,16 +674,6 @@
|
|||||||
CONST_Tablespace_Search_Index, $sTemplate);
|
CONST_Tablespace_Search_Index, $sTemplate);
|
||||||
$sTemplate = replace_tablespace('{ts:aux-index}',
|
$sTemplate = replace_tablespace('{ts:aux-index}',
|
||||||
CONST_Tablespace_Aux_Index, $sTemplate);
|
CONST_Tablespace_Aux_Index, $sTemplate);
|
||||||
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
|
|
||||||
foreach($aMatches as $aMatch)
|
|
||||||
{
|
|
||||||
$sResult = '';
|
|
||||||
foreach($aPartitions as $sPartitionName)
|
|
||||||
{
|
|
||||||
$sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
|
|
||||||
}
|
|
||||||
$sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
|
|
||||||
}
|
|
||||||
|
|
||||||
pgsqlRunScript($sTemplate);
|
pgsqlRunScript($sTemplate);
|
||||||
}
|
}
|
||||||
@@ -1011,3 +945,31 @@
|
|||||||
return $sSql;
|
return $sSql;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function create_sql_functions($aCMDResult)
|
||||||
|
{
|
||||||
|
$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
|
||||||
|
$sTemplate = str_replace('{modulepath}', CONST_InstallPath.'/module', $sTemplate);
|
||||||
|
if ($aCMDResult['enable-diff-updates'])
|
||||||
|
{
|
||||||
|
$sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
|
||||||
|
}
|
||||||
|
if ($aCMDResult['enable-debug-statements'])
|
||||||
|
{
|
||||||
|
$sTemplate = str_replace('--DEBUG:', '', $sTemplate);
|
||||||
|
}
|
||||||
|
if (CONST_Limit_Reindexing)
|
||||||
|
{
|
||||||
|
$sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate);
|
||||||
|
}
|
||||||
|
if (!CONST_Use_US_Tiger_Data)
|
||||||
|
{
|
||||||
|
$sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate);
|
||||||
|
}
|
||||||
|
if (!CONST_Use_Aux_Location_data)
|
||||||
|
{
|
||||||
|
$sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate);
|
||||||
|
}
|
||||||
|
pgsqlRunScript($sTemplate);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
433
utils/update.php
433
utils/update.php
@@ -11,18 +11,11 @@
|
|||||||
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
|
||||||
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
|
||||||
|
|
||||||
array('max-load', '', 0, 1, 1, 1, 'float', 'Maximum load average - indexing is paused if this is exceeded'),
|
|
||||||
array('max-blocking', '', 0, 1, 1, 1, 'int', 'Maximum blocking processes - indexing is aborted / paused if this is exceeded'),
|
|
||||||
|
|
||||||
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
|
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import using osmosis'),
|
||||||
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
|
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import using osmosis forever'),
|
||||||
array('no-npi', '', 0, 1, 0, 0, 'bool', 'Do not write npi index files'),
|
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolate)'),
|
||||||
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
|
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
|
||||||
|
|
||||||
array('import-npi-all', '', 0, 1, 0, 0, 'bool', 'Import npi pre-indexed files'),
|
|
||||||
|
|
||||||
array('import-hourly', '', 0, 1, 0, 0, 'bool', 'Import hourly diffs'),
|
|
||||||
array('import-daily', '', 0, 1, 0, 0, 'bool', 'Import daily diffs'),
|
|
||||||
array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
|
array('import-all', '', 0, 1, 0, 0, 'bool', 'Import all available files'),
|
||||||
|
|
||||||
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
|
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
|
||||||
@@ -37,34 +30,14 @@
|
|||||||
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
|
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
|
||||||
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
|
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
|
||||||
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
|
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
|
||||||
array('index-estrate', '', 0, 1, 1, 1, 'int', 'Estimated indexed items per second (def:30)'),
|
|
||||||
|
|
||||||
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
|
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
|
||||||
);
|
);
|
||||||
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
|
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
|
||||||
|
|
||||||
if ($aResult['import-hourly'] + $aResult['import-daily'] + isset($aResult['import-diff']) > 1)
|
|
||||||
{
|
|
||||||
showUsage($aCMDOptions, true, 'Select either import of hourly or daily');
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
|
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1;
|
||||||
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
|
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0;
|
||||||
|
|
||||||
/*
|
|
||||||
// Lock to prevent multiple copies running
|
|
||||||
if (exec('/bin/ps uww | grep '.basename(__FILE__).' | grep -v /dev/null | grep -v grep -c', $aOutput2, $iResult) > 1)
|
|
||||||
{
|
|
||||||
fail("Copy already running\n");
|
|
||||||
}
|
|
||||||
if (!isset($aResult['max-load'])) $aResult['max-load'] = 1.9;
|
|
||||||
if (!isset($aResult['max-blocking'])) $aResult['max-blocking'] = 3;
|
|
||||||
if (getBlockingProcesses() > $aResult['max-blocking'])
|
|
||||||
{
|
|
||||||
fail("Too many blocking processes for import\n");
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
date_default_timezone_set('Etc/UTC');
|
date_default_timezone_set('Etc/UTC');
|
||||||
|
|
||||||
$oDB =& getDB();
|
$oDB =& getDB();
|
||||||
@@ -86,153 +59,112 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
$bFirst = true;
|
if (isset($aResult['import-diff']))
|
||||||
$bContinue = $aResult['import-all'];
|
|
||||||
while ($bContinue || $bFirst)
|
|
||||||
{
|
{
|
||||||
$bFirst = false;
|
// import diff directly (e.g. from osmosis --rri)
|
||||||
|
$sNextFile = $aResult['import-diff'];
|
||||||
if ($aResult['import-hourly'])
|
if (!file_exists($sNextFile))
|
||||||
{
|
{
|
||||||
// Mirror the hourly diffs
|
fail("Cannot open $sNextFile\n");
|
||||||
exec('wget --quiet --mirror -l 1 -P '.$sMirrorDir.' http://planet.openstreetmap.org/hourly');
|
|
||||||
$sNextFile = $oDB->getOne('select TO_CHAR(lastimportdate,\'YYYYMMDDHH24\')||\'-\'||TO_CHAR(lastimportdate+\'1 hour\'::interval,\'YYYYMMDDHH24\')||\'.osc.gz\' from import_status');
|
|
||||||
$sNextFile = $sMirrorDir.'planet.openstreetmap.org/hourly/'.$sNextFile;
|
|
||||||
$sUpdateSQL = 'update import_status set lastimportdate = lastimportdate+\'1 hour\'::interval';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($aResult['import-daily'])
|
// Import the file
|
||||||
|
$sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
|
||||||
|
echo $sCMD."\n";
|
||||||
|
exec($sCMD, $sJunk, $iErrorLevel);
|
||||||
|
|
||||||
|
if ($iErrorLevel)
|
||||||
{
|
{
|
||||||
// Mirror the daily diffs
|
fail("Error from osm2pgsql, $iErrorLevel\n");
|
||||||
exec('wget --quiet --mirror -l 1 -P '.$sMirrorDir.' http://planet.openstreetmap.org/daily');
|
|
||||||
$sNextFile = $oDB->getOne('select TO_CHAR(lastimportdate,\'YYYYMMDD\')||\'-\'||TO_CHAR(lastimportdate+\'1 day\'::interval,\'YYYYMMDD\')||\'.osc.gz\' from import_status');
|
|
||||||
$sNextFile = $sMirrorDir.'planet.openstreetmap.org/daily/'.$sNextFile;
|
|
||||||
$sUpdateSQL = 'update import_status set lastimportdate = lastimportdate::date + 1';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (isset($aResult['import-diff']))
|
|
||||||
{
|
|
||||||
// import diff directly (e.g. from osmosis --rri)
|
|
||||||
$sNextFile = $aResult['import-diff'];
|
|
||||||
if (!file_exists($sNextFile))
|
|
||||||
{
|
|
||||||
fail("Cannot open $sNextFile\n");
|
|
||||||
}
|
|
||||||
// Don't update the import status - we don't know what this file contains
|
|
||||||
$sUpdateSQL = 'update import_status set lastimportdate = now() where false';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Missing file is not an error - it might not be created yet
|
// Don't update the import status - we don't know what this file contains
|
||||||
if (($aResult['import-hourly'] || $aResult['import-daily'] || isset($aResult['import-diff'])) && file_exists($sNextFile))
|
|
||||||
{
|
|
||||||
// Import the file
|
|
||||||
$sCMD = $sOsm2pgsqlCmd.' '.$sNextFile;
|
|
||||||
echo $sCMD."\n";
|
|
||||||
exec($sCMD, $sJunk, $iErrorLevel);
|
|
||||||
|
|
||||||
if ($iErrorLevel)
|
|
||||||
{
|
|
||||||
fail("Error from osm2pgsql, $iErrorLevel\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Move the date onwards
|
|
||||||
$oDB->query($sUpdateSQL);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
$bContinue = false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
$bModifyXML = false;
|
$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
|
||||||
$sModifyXMLstr = '';
|
$bHaveDiff = false;
|
||||||
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
|
|
||||||
if (isset($aResult['import-file']) && $aResult['import-file'])
|
if (isset($aResult['import-file']) && $aResult['import-file'])
|
||||||
{
|
{
|
||||||
$bModifyXML = true;
|
$bHaveDiff = true;
|
||||||
|
$sCMD = CONST_Osmosis_Binary.' --read-xml \''.$aResult['import-file'].'\' --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
|
||||||
|
echo $sCMD."\n";
|
||||||
|
exec($sCMD, $sJunk, $iErrorLevel);
|
||||||
|
if ($iErrorLevel)
|
||||||
|
{
|
||||||
|
fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
|
||||||
|
$sContentURL = '';
|
||||||
if (isset($aResult['import-node']) && $aResult['import-node'])
|
if (isset($aResult['import-node']) && $aResult['import-node'])
|
||||||
{
|
{
|
||||||
$bModifyXML = true;
|
|
||||||
if ($bUseOSMApi)
|
if ($bUseOSMApi)
|
||||||
{
|
{
|
||||||
$sModifyXMLstr = file_get_contents('http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node']);
|
$sContentURL = 'http://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node'];
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
$sModifyXMLstr = file_get_contents('http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;');
|
$sContentURL = 'http://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (isset($aResult['import-way']) && $aResult['import-way'])
|
if (isset($aResult['import-way']) && $aResult['import-way'])
|
||||||
{
|
{
|
||||||
$bModifyXML = true;
|
|
||||||
if ($bUseOSMApi)
|
if ($bUseOSMApi)
|
||||||
{
|
{
|
||||||
$sCmd = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
|
$sContentURL = 'http://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full';
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
$sCmd = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
|
$sContentURL = 'http://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;';
|
||||||
}
|
}
|
||||||
$sModifyXMLstr = file_get_contents($sCmd);
|
|
||||||
}
|
}
|
||||||
if (isset($aResult['import-relation']) && $aResult['import-relation'])
|
if (isset($aResult['import-relation']) && $aResult['import-relation'])
|
||||||
{
|
{
|
||||||
$bModifyXML = true;
|
|
||||||
if ($bUseOSMApi)
|
if ($bUseOSMApi)
|
||||||
{
|
{
|
||||||
$sModifyXMLstr = file_get_contents('http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full');
|
$sContentURLsModifyXMLstr = 'http://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full';
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
$sModifyXMLstr = file_get_contents('http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;');
|
$sContentURL = 'http://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ($bModifyXML)
|
if ($sContentURL)
|
||||||
{
|
{
|
||||||
// derive change from normal osm file with osmosis
|
$sModifyXMLstr = file_get_contents($sContentURL);
|
||||||
$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc';
|
$bHaveDiff = true;
|
||||||
if (isset($aResult['import-file']) && $aResult['import-file'])
|
|
||||||
{
|
|
||||||
$sCMD = CONST_Osmosis_Binary.' --read-xml \''.$aResult['import-file'].'\' --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
|
|
||||||
echo $sCMD."\n";
|
|
||||||
exec($sCMD, $sJunk, $iErrorLevel);
|
|
||||||
if ($iErrorLevel)
|
|
||||||
{
|
|
||||||
fail("Error converting osm to osc, osmosis returned: $iErrorLevel\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
$aSpec = array(
|
|
||||||
0 => array("pipe", "r"), // stdin
|
|
||||||
1 => array("pipe", "w"), // stdout
|
|
||||||
2 => array("pipe", "w") // stderr
|
|
||||||
);
|
|
||||||
$sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
|
|
||||||
echo $sCMD."\n";
|
|
||||||
$hProc = proc_open($sCMD, $aSpec, $aPipes);
|
|
||||||
if (!is_resource($hProc))
|
|
||||||
{
|
|
||||||
fail("Error converting osm to osc, osmosis failed\n");
|
|
||||||
}
|
|
||||||
fwrite($aPipes[0], $sModifyXMLstr);
|
|
||||||
fclose($aPipes[0]);
|
|
||||||
$sOut = stream_get_contents($aPipes[1]);
|
|
||||||
if ($aResult['verbose']) echo $sOut;
|
|
||||||
fclose($aPipes[1]);
|
|
||||||
$sErrors = stream_get_contents($aPipes[2]);
|
|
||||||
if ($aResult['verbose']) echo $sErrors;
|
|
||||||
fclose($aPipes[2]);
|
|
||||||
if ($iError = proc_close($hProc))
|
|
||||||
{
|
|
||||||
echo "Error converting osm to osc, osmosis returned: $iError\n";
|
|
||||||
echo $sOut;
|
|
||||||
echo $sErrors;
|
|
||||||
exit(-1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
$aSpec = array(
|
||||||
|
0 => array("pipe", "r"), // stdin
|
||||||
|
1 => array("pipe", "w"), // stdout
|
||||||
|
2 => array("pipe", "w") // stderr
|
||||||
|
);
|
||||||
|
$sCMD = CONST_Osmosis_Binary.' --read-xml - --read-empty --derive-change --write-xml-change '.$sTemporaryFile;
|
||||||
|
echo $sCMD."\n";
|
||||||
|
$hProc = proc_open($sCMD, $aSpec, $aPipes);
|
||||||
|
if (!is_resource($hProc))
|
||||||
|
{
|
||||||
|
fail("Error converting osm to osc, osmosis failed\n");
|
||||||
|
}
|
||||||
|
fwrite($aPipes[0], $sModifyXMLstr);
|
||||||
|
fclose($aPipes[0]);
|
||||||
|
$sOut = stream_get_contents($aPipes[1]);
|
||||||
|
if ($aResult['verbose']) echo $sOut;
|
||||||
|
fclose($aPipes[1]);
|
||||||
|
$sErrors = stream_get_contents($aPipes[2]);
|
||||||
|
if ($aResult['verbose']) echo $sErrors;
|
||||||
|
fclose($aPipes[2]);
|
||||||
|
if ($iError = proc_close($hProc))
|
||||||
|
{
|
||||||
|
echo $sOut;
|
||||||
|
echo $sErrors;
|
||||||
|
fail("Error converting osm to osc, osmosis returned: $iError\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($bHaveDiff)
|
||||||
|
{
|
||||||
// import generated change file
|
// import generated change file
|
||||||
$sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
|
$sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile;
|
||||||
echo $sCMD."\n";
|
echo $sCMD."\n";
|
||||||
@@ -246,19 +178,19 @@
|
|||||||
if ($aResult['deduplicate'])
|
if ($aResult['deduplicate'])
|
||||||
{
|
{
|
||||||
|
|
||||||
$pgver = (float) CONST_Postgresql_Version;
|
if (getPostgresVersion() < 9.3)
|
||||||
if ($pgver < 9.3) {
|
{
|
||||||
fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
|
fail("ERROR: deduplicate is only currently supported in postgresql 9.3");
|
||||||
}
|
}
|
||||||
|
|
||||||
$oDB =& getDB();
|
$oDB =& getDB();
|
||||||
$sSQL = 'select partition from country_name order by country_code';
|
$sSQL = 'select partition from country_name order by country_code';
|
||||||
$aPartitions = $oDB->getCol($sSQL);
|
$aPartitions = $oDB->getCol($sSQL);
|
||||||
if (PEAR::isError($aPartitions))
|
if (PEAR::isError($aPartitions))
|
||||||
{
|
{
|
||||||
fail($aPartitions->getMessage());
|
fail($aPartitions->getMessage());
|
||||||
}
|
}
|
||||||
$aPartitions[] = 0;
|
$aPartitions[] = 0;
|
||||||
|
|
||||||
$sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
|
$sSQL = "select word_token,count(*) from word where substr(word_token, 1, 1) = ' ' and class is null and type is null and country_code is null group by word_token having count(*) > 1 order by word_token";
|
||||||
$aDuplicateTokens = $oDB->getAll($sSQL);
|
$aDuplicateTokens = $oDB->getAll($sSQL);
|
||||||
@@ -341,7 +273,6 @@
|
|||||||
exit(1);
|
exit(1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -358,92 +289,81 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
$sImportFile = CONST_BasePath.'/data/osmosischange.osc';
|
$sImportFile = CONST_BasePath.'/data/osmosischange.osc';
|
||||||
$sOsmosisCMD = CONST_Osmosis_Binary;
|
|
||||||
$sOsmosisConfigDirectory = CONST_InstallPath.'/settings';
|
$sOsmosisConfigDirectory = CONST_InstallPath.'/settings';
|
||||||
$sCMDDownload = $sOsmosisCMD.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
|
$sCMDDownload = CONST_Osmosis_Binary.' --read-replication-interval workingDirectory='.$sOsmosisConfigDirectory.' --simplify-change --write-xml-change '.$sImportFile;
|
||||||
$sCMDCheckReplicationLag = $sOsmosisCMD.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
|
$sCMDCheckReplicationLag = CONST_Osmosis_Binary.' -q --read-replication-lag workingDirectory='.$sOsmosisConfigDirectory;
|
||||||
$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
|
$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile;
|
||||||
$sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
|
$sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'];
|
||||||
if (!$aResult['no-npi']) {
|
|
||||||
$sCMDIndex .= '-F ';
|
|
||||||
}
|
|
||||||
while(true)
|
while(true)
|
||||||
{
|
{
|
||||||
$fStartTime = time();
|
$fStartTime = time();
|
||||||
$iFileSize = 1001;
|
$iFileSize = 1001;
|
||||||
|
|
||||||
// Logic behind this is that osm2pgsql locks the database quite a bit
|
if (!file_exists($sImportFile))
|
||||||
// So it is better to import lots of small files
|
{
|
||||||
// But indexing works most efficiently on large amounts of data
|
// First check if there are new updates published (except for minutelies - there's always new diffs to process)
|
||||||
// So do lots of small imports and a BIG index
|
if ( CONST_Replication_Update_Interval > 60 )
|
||||||
|
|
||||||
// while($aResult['import-osmosis-all'] && $iFileSize > 1000)
|
|
||||||
// {
|
|
||||||
if (!file_exists($sImportFile))
|
|
||||||
{
|
{
|
||||||
// First check if there are new updates published (except for minutelies - there's always new diffs to process)
|
|
||||||
if ( CONST_Replication_Update_Interval > 60 )
|
|
||||||
{
|
|
||||||
|
|
||||||
|
unset($aReplicationLag);
|
||||||
|
exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
|
||||||
|
while ($iErrorLevel > 0 || $aReplicationLag[0] < 1)
|
||||||
|
{
|
||||||
|
if ($iErrorLevel)
|
||||||
|
{
|
||||||
|
echo "Error: $iErrorLevel. ";
|
||||||
|
echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
echo ".";
|
||||||
|
}
|
||||||
|
sleep(CONST_Replication_Recheck_Interval);
|
||||||
unset($aReplicationLag);
|
unset($aReplicationLag);
|
||||||
exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
|
exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
|
||||||
while ($iErrorLevel > 0 || $aReplicationLag[0] < 1)
|
|
||||||
{
|
|
||||||
if ($iErrorLevel)
|
|
||||||
{
|
|
||||||
echo "Error: $iErrorLevel. ";
|
|
||||||
echo "Re-trying: ".$sCMDCheckReplicationLag." in ".CONST_Replication_Recheck_Interval." secs\n";
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
echo ".";
|
|
||||||
}
|
|
||||||
sleep(CONST_Replication_Recheck_Interval);
|
|
||||||
unset($aReplicationLag);
|
|
||||||
exec($sCMDCheckReplicationLag, $aReplicationLag, $iErrorLevel);
|
|
||||||
}
|
|
||||||
// There are new replication files - use osmosis to download the file
|
|
||||||
echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
|
|
||||||
}
|
}
|
||||||
$fStartTime = time();
|
// There are new replication files - use osmosis to download the file
|
||||||
$fCMDStartTime = time();
|
echo "\n".date('Y-m-d H:i:s')." Replication Delay is ".$aReplicationLag[0]."\n";
|
||||||
echo $sCMDDownload."\n";
|
|
||||||
exec($sCMDDownload, $sJunk, $iErrorLevel);
|
|
||||||
while ($iErrorLevel > 0)
|
|
||||||
{
|
|
||||||
echo "Error: $iErrorLevel\n";
|
|
||||||
sleep(60);
|
|
||||||
echo 'Re-trying: '.$sCMDDownload."\n";
|
|
||||||
exec($sCMDDownload, $sJunk, $iErrorLevel);
|
|
||||||
}
|
|
||||||
$iFileSize = filesize($sImportFile);
|
|
||||||
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
|
||||||
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
|
|
||||||
var_Dump($sSQL);
|
|
||||||
$oDB->query($sSQL);
|
|
||||||
echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
|
||||||
}
|
}
|
||||||
|
$fStartTime = time();
|
||||||
$iFileSize = filesize($sImportFile);
|
|
||||||
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
|
||||||
|
|
||||||
// Import the file
|
|
||||||
$fCMDStartTime = time();
|
$fCMDStartTime = time();
|
||||||
echo $sCMDImport."\n";
|
echo $sCMDDownload."\n";
|
||||||
exec($sCMDImport, $sJunk, $iErrorLevel);
|
exec($sCMDDownload, $sJunk, $iErrorLevel);
|
||||||
if ($iErrorLevel)
|
while ($iErrorLevel > 0)
|
||||||
{
|
{
|
||||||
echo "Error: $iErrorLevel\n";
|
echo "Error: $iErrorLevel\n";
|
||||||
exit($iErrorLevel);
|
sleep(60);
|
||||||
|
echo 'Re-trying: '.$sCMDDownload."\n";
|
||||||
|
exec($sCMDDownload, $sJunk, $iErrorLevel);
|
||||||
}
|
}
|
||||||
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
|
$iFileSize = filesize($sImportFile);
|
||||||
|
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
||||||
|
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osmosis')";
|
||||||
var_Dump($sSQL);
|
var_Dump($sSQL);
|
||||||
$oDB->query($sSQL);
|
$oDB->query($sSQL);
|
||||||
echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
echo date('Y-m-d H:i:s')." Completed osmosis step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
||||||
|
}
|
||||||
|
|
||||||
// Archive for debug?
|
$iFileSize = filesize($sImportFile);
|
||||||
unlink($sImportFile);
|
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
||||||
// }
|
|
||||||
|
// Import the file
|
||||||
|
$fCMDStartTime = time();
|
||||||
|
echo $sCMDImport."\n";
|
||||||
|
exec($sCMDImport, $sJunk, $iErrorLevel);
|
||||||
|
if ($iErrorLevel)
|
||||||
|
{
|
||||||
|
echo "Error: $iErrorLevel\n";
|
||||||
|
exit($iErrorLevel);
|
||||||
|
}
|
||||||
|
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','osm2pgsql')";
|
||||||
|
var_Dump($sSQL);
|
||||||
|
$oDB->query($sSQL);
|
||||||
|
echo date('Y-m-d H:i:s')." Completed osm2pgsql step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
||||||
|
|
||||||
|
// Archive for debug?
|
||||||
|
unlink($sImportFile);
|
||||||
|
|
||||||
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
$sBatchEnd = getosmosistimestamp($sOsmosisConfigDirectory);
|
||||||
|
|
||||||
@@ -451,31 +371,6 @@
|
|||||||
$sThisIndexCmd = $sCMDIndex;
|
$sThisIndexCmd = $sCMDIndex;
|
||||||
$fCMDStartTime = time();
|
$fCMDStartTime = time();
|
||||||
|
|
||||||
if (!$aResult['no-npi'])
|
|
||||||
{
|
|
||||||
$iFileID = $oDB->getOne('select nextval(\'file\')');
|
|
||||||
if (PEAR::isError($iFileID))
|
|
||||||
{
|
|
||||||
echo $iFileID->getMessage()."\n";
|
|
||||||
exit(-1);
|
|
||||||
}
|
|
||||||
$sFileDir = CONST_BasePath.'/export/diff/';
|
|
||||||
$sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT);
|
|
||||||
$sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT);
|
|
||||||
|
|
||||||
if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true);
|
|
||||||
$sThisIndexCmd .= $sFileDir;
|
|
||||||
$sThisIndexCmd .= '/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT);
|
|
||||||
$sThisIndexCmd .= ".npi.out";
|
|
||||||
|
|
||||||
preg_match('#^([0-9]{4})-([0-9]{2})-([0-9]{2})#', $sBatchEnd, $aBatchMatch);
|
|
||||||
$sFileDir = CONST_BasePath.'/export/index/';
|
|
||||||
$sFileDir .= $aBatchMatch[1].'/'.$aBatchMatch[2];
|
|
||||||
|
|
||||||
if (!is_dir($sFileDir)) mkdir($sFileDir, 0777, true);
|
|
||||||
file_put_contents($sFileDir.'/'.$aBatchMatch[3].'.idx', "$sBatchEnd\t$iFileID\n", FILE_APPEND);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!$aResult['no-index'])
|
if (!$aResult['no-index'])
|
||||||
{
|
{
|
||||||
echo "$sThisIndexCmd\n";
|
echo "$sThisIndexCmd\n";
|
||||||
@@ -485,25 +380,6 @@
|
|||||||
echo "Error: $iErrorLevel\n";
|
echo "Error: $iErrorLevel\n";
|
||||||
exit($iErrorLevel);
|
exit($iErrorLevel);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!$aResult['no-npi'])
|
|
||||||
{
|
|
||||||
$sFileDir = CONST_BasePath.'/export/diff/';
|
|
||||||
$sFileDir .= str_pad(floor($iFileID/1000000), 3, '0', STR_PAD_LEFT);
|
|
||||||
$sFileDir .= '/'.str_pad(floor($iFileID/1000) % 1000, 3, '0', STR_PAD_LEFT);
|
|
||||||
|
|
||||||
$sThisIndexCmd = 'bzip2 -z9 '.$sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out";
|
|
||||||
echo "$sThisIndexCmd\n";
|
|
||||||
exec($sThisIndexCmd, $sJunk, $iErrorLevel);
|
|
||||||
if ($iErrorLevel)
|
|
||||||
{
|
|
||||||
echo "Error: $iErrorLevel\n";
|
|
||||||
exit($iErrorLevel);
|
|
||||||
}
|
|
||||||
|
|
||||||
rename($sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.out.bz2",
|
|
||||||
$sFileDir.'/'.str_pad($iFileID % 1000, 3, '0', STR_PAD_LEFT).".npi.bz2");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
|
$sSQL = "INSERT INTO import_osmosis_log values ('$sBatchEnd',$iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','index')";
|
||||||
@@ -529,55 +405,6 @@
|
|||||||
echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
|
echo date('Y-m-d H:i:s')." Sleeping $iSleep seconds\n";
|
||||||
sleep($iSleep);
|
sleep($iSleep);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($aResult['import-npi-all'])
|
|
||||||
{
|
|
||||||
$iNPIID = $oDB->getOne('select max(npiid) from import_npi_log');
|
|
||||||
if (PEAR::isError($iNPIID))
|
|
||||||
{
|
|
||||||
var_dump($iNPIID);
|
|
||||||
exit(1);
|
|
||||||
}
|
|
||||||
$sConfigDirectory = CONST_InstallPath.'/settings';
|
|
||||||
$sCMDImportTemplate = CONST_InstallPath.'/nominatim/nominatim -d gazetteer -P 5433 -I -T '.CONST_BasePath.'/settings/partitionedtags.def -F ';
|
|
||||||
while(true)
|
|
||||||
{
|
|
||||||
$fStartTime = time();
|
|
||||||
|
|
||||||
$iNPIID++;
|
|
||||||
|
|
||||||
$sImportFile = CONST_BasePath.'/export/diff/';
|
|
||||||
$sImportFile .= str_pad(floor($iNPIID/1000000), 3, '0', STR_PAD_LEFT);
|
|
||||||
$sImportFile .= '/'.str_pad(floor($iNPIID/1000) % 1000, 3, '0', STR_PAD_LEFT);
|
|
||||||
$sImportFile .= '/'.str_pad($iNPIID % 1000, 3, '0', STR_PAD_LEFT);
|
|
||||||
$sImportFile .= ".npi";
|
|
||||||
while(!file_exists($sImportFile) && !file_exists($sImportFile.'.bz2'))
|
|
||||||
{
|
|
||||||
echo "sleep (waiting for $sImportFile)\n";
|
|
||||||
sleep(10);
|
|
||||||
}
|
|
||||||
if (file_exists($sImportFile.'.bz2')) $sImportFile .= '.bz2';
|
|
||||||
|
|
||||||
$iFileSize = filesize($sImportFile);
|
|
||||||
|
|
||||||
// Import the file
|
|
||||||
$fCMDStartTime = time();
|
|
||||||
$sCMDImport = $sCMDImportTemplate . $sImportFile;
|
|
||||||
echo $sCMDImport."\n";
|
|
||||||
exec($sCMDImport, $sJunk, $iErrorLevel);
|
|
||||||
if ($iErrorLevel)
|
|
||||||
{
|
|
||||||
fail("Error: $iErrorLevel\n");
|
|
||||||
}
|
|
||||||
$sBatchEnd = $iNPIID;
|
|
||||||
echo "Completed for $sBatchEnd in ".round((time()-$fCMDStartTime)/60,2)." minutes\n";
|
|
||||||
$sSQL = "INSERT INTO import_npi_log values ($iNPIID, null, $iFileSize,'".date('Y-m-d H:i:s',$fCMDStartTime)."','".date('Y-m-d H:i:s')."','import')";
|
|
||||||
var_Dump($sSQL);
|
|
||||||
$oDB->query($sSQL);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function getosmosistimestamp($sOsmosisConfigDirectory)
|
function getosmosistimestamp($sOsmosisConfigDirectory)
|
||||||
|
|||||||
@@ -56,10 +56,17 @@
|
|||||||
|
|
||||||
$iPlaceID = (int)$_GET['place_id'];
|
$iPlaceID = (int)$_GET['place_id'];
|
||||||
|
|
||||||
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
|
if (CONST_Use_US_Tiger_Data)
|
||||||
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
|
{
|
||||||
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
|
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
|
||||||
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
|
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (CONST_Use_Aux_Location_data)
|
||||||
|
{
|
||||||
|
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
|
||||||
|
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
|
||||||
|
}
|
||||||
|
|
||||||
$hLog = logStart($oDB, 'details', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
|
$hLog = logStart($oDB, 'details', $_SERVER['QUERY_STRING'], $aLangPrefOrder);
|
||||||
|
|
||||||
|
|||||||
@@ -47,13 +47,13 @@
|
|||||||
|
|
||||||
$iPlaceID = (int)$_GET['place_id'];
|
$iPlaceID = (int)$_GET['place_id'];
|
||||||
|
|
||||||
$sAuxHouseNumber = false;
|
if (CONST_Use_US_Tiger_Data)
|
||||||
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
|
|
||||||
if ($iParentPlaceID)
|
|
||||||
{
|
{
|
||||||
$iPlaceID = $iParentPlaceID;
|
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_tiger where place_id = '.$iPlaceID);
|
||||||
|
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
|
||||||
}
|
}
|
||||||
else
|
|
||||||
|
if (CONST_Use_Aux_Location_data)
|
||||||
{
|
{
|
||||||
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
|
$iParentPlaceID = $oDB->getOne('select parent_place_id from location_property_aux where place_id = '.$iPlaceID);
|
||||||
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
|
if ($iParentPlaceID) $iPlaceID = $iParentPlaceID;
|
||||||
|
|||||||
Reference in New Issue
Block a user