hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-16 15:47:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove interpolation lines from placex and save them in an extra table.

Browse Source
This commit is contained in:
Markus Gail
2016-04-25 09:44:01 +02:00
parent 0419aada6e
commit 7879ad44cd
19 changed files with 1627 additions and 1155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
lib/Geocode.php
View File

@@ -433,21 +433,44 @@
if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank) if (30 >= $this->iMinAddressRank && 30 <= $this->iMaxAddressRank)
{ {
//query also location_property_tiger and location_property_aux //query also location_property_osmline and location_property_tiger and location_property_aux
//Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join) //Tiger search only if a housenumber was searched and if it was found (i.e. aPlaceIDs[placeID] = housenumber != -1) (realized through a join)
//only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points //only Tiger housenumbers need to be interpolated, because they are saved as lines with start- and endnumber, the common osm housenumbers are usually saved as points
$sHousenumbers = ""; $sHousenumbers = "";
$i = 0; $i = 0;
$length = count($aPlaceIDs); $length = count($aPlaceIDs);
foreach($aPlaceIDs as $placeID => $housenumber) foreach($aPlaceIDs as $placeID => $housenumber)
{ {
$i++; $i++;
$sHousenumbers .= "(".$placeID.", ".$housenumber.")"; $sHousenumbers .= "(".$placeID.", ".$housenumber.")";
if($i<$length) if($i<$length)
$sHousenumbers .= ", "; $sHousenumbers .= ", ";
} }
// osmline, osm_type is 'I' for Interpolation Line
$sSQL .= " union ";
$sSQL .= "select 'I' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, calculated_country_code as country_code, ";
$sSQL .= "get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress, ";
$sSQL .= "null as placename, ";
$sSQL .= "null as ref, ";
if ($this->bIncludeExtraTags) $sSQL .= "null as extra, ";
if ($this->bIncludeNameDetails) $sSQL .= "null as names, ";
$sSQL .= " avg(st_x(centroid)) as lon, avg(st_y(centroid)) as lat,";
$sSQL .= $sImportanceSQL."-0.1 as importance, "; // slightly smaller than the importance for normal houses with rank 30, which is 0
$sSQL .= " (select max(p.importance*(p.rank_address+2)) from place_addressline s, placex p where s.place_id = min(blub.parent_place_id) and p.place_id = s.address_place_id and s.isaddress and p.importance is not null) as addressimportance, ";
$sSQL .= " null as extra_place ";
$sSQL .= " from (select place_id, calculated_country_code ";
//interpolate the housenumbers here
$sSQL .= ", CASE WHEN startnumber != endnumber THEN ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) ";
$sSQL .= " ELSE ST_LineInterpolatePoint(linegeo, 0.5) END as centroid";
$sSQL .= ", parent_place_id, housenumber_for_place ";
$sSQL .= " from (location_property_osmline ";
$sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
$sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
$sSQL .= " group by place_id, housenumber_for_place, calculated_country_code "; //is this group by really needed?, place_id + housenumber (in combination) are unique
if (!$this->bDeDupe) $sSQL .= ", place_id ";
$sSQL .= "union "; // tiger
$sSQL .= " union ";
$sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code"; $sSQL .= "select 'T' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 30 as rank_search, 30 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code";
$sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress "; $sSQL .= ", get_address_by_language(place_id, housenumber_for_place, $sLanguagePrefArraySQL) as langaddress ";
$sSQL .= ", null as placename"; $sSQL .= ", null as placename";
@@ -460,13 +483,16 @@
$sSQL .= ", null as extra_place "; $sSQL .= ", null as extra_place ";
$sSQL .= " from (select place_id"; $sSQL .= " from (select place_id";
//interpolate the Tiger housenumbers here //interpolate the Tiger housenumbers here
$sSQL .= ", ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) as centroid, parent_place_id, housenumber_for_place "; $sSQL .= ", CASE WHEN startnumber != endnumber THEN ST_LineInterpolatePoint(linegeo, (housenumber_for_place-startnumber::float)/(endnumber-startnumber)::float) ";
$sSQL .= "from (location_property_tiger "; $sSQL .= " ELSE ST_LineInterpolatePoint(linegeo, 0.5) END as centroid";
$sSQL .= ", parent_place_id, housenumber_for_place ";
$sSQL .= " from (location_property_tiger ";
$sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) "; $sSQL .= " join (values ".$sHousenumbers.") as housenumbers(place_id, housenumber_for_place) using(place_id)) ";
$sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here $sSQL .= " where housenumber_for_place>=0 and 30 between $this->iMinAddressRank and $this->iMaxAddressRank) as blub"; //postgres wants an alias here
$sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique $sSQL .= " group by place_id, housenumber_for_place"; //is this group by really needed?, place_id + housenumber (in combination) are unique
if (!$this->bDeDupe) $sSQL .= ", place_id "; if (!$this->bDeDupe) $sSQL .= ", place_id ";
// aux
$sSQL .= " union "; $sSQL .= " union ";
$sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, "; $sSQL .= "select 'L' as osm_type, place_id as osm_id, 'place' as class, 'house' as type, null as admin_level, 0 as rank_search, 0 as rank_address, min(place_id) as place_id, min(parent_place_id) as parent_place_id, 'us' as country_code, ";
$sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, "; $sSQL .= "get_address_by_language(place_id, -1, $sLanguagePrefArraySQL) as langaddress, ";
@@ -875,7 +901,7 @@
// Do we have anything that looks like a lat/lon pair? // Do we have anything that looks like a lat/lon pair?
if ( $aLooksLike = looksLikeLatLonPair($sQuery) ) if ( $aLooksLike = looksLikeLatLonPair($sQuery) )
{ {
$this->setNearPoint(array($aLooksLike['lat'], $aLooksLike['lon'])); $this->setNearPoint(array($aLooksLike['lat'], $aLooksLike['lon']));
$sQuery = $aLooksLike['query']; $sQuery = $aLooksLike['query'];
} }
@@ -1303,7 +1329,11 @@
if ($aSearch['sHouseNumber'] && sizeof($aSearch['aAddress'])) if ($aSearch['sHouseNumber'] && sizeof($aSearch['aAddress']))
{ {
$sHouseNumberRegex = '\\\\m'.$aSearch['sHouseNumber'].'\\\\M'; $sHouseNumberRegex = '\\\\m'.$aSearch['sHouseNumber'].'\\\\M';
$aOrder[] = "exists(select place_id from placex where parent_place_id = search_name.place_id and transliteration(housenumber) ~* E'".$sHouseNumberRegex."' limit 1) desc"; $aOrder[] = "";
$aOrder[0] = "exists(select place_id from placex where parent_place_id = search_name.place_id and transliteration(housenumber) ~* E'".$sHouseNumberRegex."' limit 1) ";
// also housenumbers from interpolation lines table are needed
$aOrder[0] .= " or exists(select place_id from location_property_osmline where parent_place_id = search_name.place_id and ".$aSearch['sHouseNumber'].">=startnumber and ".$aSearch['sHouseNumber']."<=endnumber limit 1)";
$aOrder[0] .= " desc";
} }
// TODO: filter out the pointless search terms (2 letter name tokens and less) // TODO: filter out the pointless search terms (2 letter name tokens and less)
@@ -1418,6 +1448,7 @@
//now search for housenumber, if housenumber provided //now search for housenumber, if housenumber provided
if ($aSearch['sHouseNumber'] && sizeof($aPlaceIDs)) if ($aSearch['sHouseNumber'] && sizeof($aPlaceIDs))
{ {
$searchedHousenumber = intval($aSearch['sHouseNumber']);
$aRoadPlaceIDs = $aPlaceIDs; $aRoadPlaceIDs = $aPlaceIDs;
$sPlaceIDs = join(',',$aPlaceIDs); $sPlaceIDs = join(',',$aPlaceIDs);
@@ -1432,20 +1463,42 @@
if (CONST_Debug) var_dump($sSQL); if (CONST_Debug) var_dump($sSQL);
$aPlaceIDs = $this->oDB->getCol($sSQL); $aPlaceIDs = $this->oDB->getCol($sSQL);
// if nothing found, search in the interpolation line table
if(!sizeof($aPlaceIDs))
{
// do we need to use transliteration and the regex for housenumbers???
//new query for lines, not housenumbers anymore
if($searchedHousenumber%2 == 0){
//if housenumber is even, look for housenumber in streets with interpolationtype even or all
$sSQL = "select distinct place_id from location_property_osmline where parent_place_id in (".$sPlaceIDs.") and (interpolationtype='even' or interpolationtype='all') and ".$searchedHousenumber.">=startnumber and ".$searchedHousenumber."<=endnumber";
}else{
//look for housenumber in streets with interpolationtype odd or all
$sSQL = "select distinct place_id from location_property_osmline where parent_place_id in (".$sPlaceIDs.") and (interpolationtype='odd' or interpolationtype='all') and ".$searchedHousenumber.">=startnumber and ".$searchedHousenumber."<=endnumber";
}
if (sizeof($this->aExcludePlaceIDs))
{
$sSQL .= " and parent_place_id not in (".join(',', $this->aExcludePlaceIDs).")";
}
//$sSQL .= " limit $this->iLimit";
if (CONST_Debug) var_dump($sSQL);
//get place IDs
$aPlaceIDs = $this->oDB->getCol($sSQL, 0);
}
// If nothing found try the aux fallback table // If nothing found try the aux fallback table
if (!sizeof($aPlaceIDs)) if (!sizeof($aPlaceIDs))
{ {
$sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'"; $sSQL = "select place_id from location_property_aux where parent_place_id in (".$sPlaceIDs.") and housenumber = '".pg_escape_string($aSearch['sHouseNumber'])."'";
if (sizeof($this->aExcludePlaceIDs)) if (sizeof($this->aExcludePlaceIDs))
{ {
$sSQL .= " and place_id not in (".join(',',$this->aExcludePlaceIDs).")"; $sSQL .= " and parent_place_id not in (".join(',',$this->aExcludePlaceIDs).")";
} }
//$sSQL .= " limit $this->iLimit"; //$sSQL .= " limit $this->iLimit";
if (CONST_Debug) var_dump($sSQL); if (CONST_Debug) var_dump($sSQL);
$aPlaceIDs = $this->oDB->getCol($sSQL); $aPlaceIDs = $this->oDB->getCol($sSQL);
} }
//if nothing was found in placex or location_property_aux, then search in Tiger data for this housenumber(location_property_tiger) //if nothing found search in Tiger data for this housenumber(location_property_tiger)
$searchedHousenumber = intval($aSearch['sHouseNumber']);
if (!sizeof($aPlaceIDs)) if (!sizeof($aPlaceIDs))
{ {
//new query for lines, not housenumbers anymore //new query for lines, not housenumbers anymore
@@ -1614,6 +1667,7 @@
{ {
// Need to verify passes rank limits before dropping out of the loop (yuk!) // Need to verify passes rank limits before dropping out of the loop (yuk!)
// reduces the number of place ids, like a filter // reduces the number of place ids, like a filter
// rank_address is 30 for interpolated housenumbers
$sSQL = "select place_id from placex where place_id in (".join(',',array_keys($aResultPlaceIDs)).") "; $sSQL = "select place_id from placex where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
$sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank "; $sSQL .= "and (placex.rank_address between $this->iMinAddressRank and $this->iMaxAddressRank ";
if (14 >= $this->iMinAddressRank && 14 <= $this->iMaxAddressRank) $sSQL .= " OR (extratags->'place') = 'city'"; if (14 >= $this->iMinAddressRank && 14 <= $this->iMaxAddressRank) $sSQL .= " OR (extratags->'place') = 'city'";
@@ -1621,7 +1675,8 @@
$sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") "; $sSQL .= ") UNION select place_id from location_property_tiger where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
$sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank "; $sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank ";
if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")"; if ($this->aAddressRankList) $sSQL .= " OR 30 in (".join(',',$this->aAddressRankList).")";
$sSQL .= ")"; $sSQL .= ") UNION select place_id from location_property_osmline where place_id in (".join(',',array_keys($aResultPlaceIDs)).") ";
$sSQL .= "and (30 between $this->iMinAddressRank and $this->iMaxAddressRank)";
if (CONST_Debug) var_dump($sSQL); if (CONST_Debug) var_dump($sSQL);
$aFilteredPlaceIDs = $this->oDB->getCol($sSQL); $aFilteredPlaceIDs = $this->oDB->getCol($sSQL);
$tempIDs = array(); $tempIDs = array();

31
lib/PlaceLookup.php
View File

@@ -117,7 +117,7 @@
{ {
$this->setOSMID($details['osm_type'], $details['osm_id']); $this->setOSMID($details['osm_type'], $details['osm_id']);
} }
if (isset($details['fraction'])) $this->fTigerFraction = $details['fraction']; if (isset($details['fraction'])) $this->fInterpolFraction = $details['fraction'];
return $this->lookup(); return $this->lookup();
} }
@@ -140,12 +140,33 @@
if ($this->bNameDetails) $sSQL .= " null as names,"; if ($this->bNameDetails) $sSQL .= " null as names,";
$sSQL .= " ST_X(point) as lon, ST_Y(point) as lat from (select *, ST_LineInterpolatePoint(linegeo, (housenumber-startnumber::float)/(endnumber-startnumber)::float) as point from "; $sSQL .= " ST_X(point) as lon, ST_Y(point) as lat from (select *, ST_LineInterpolatePoint(linegeo, (housenumber-startnumber::float)/(endnumber-startnumber)::float) as point from ";
$sSQL .= " (select *, "; $sSQL .= " (select *, ";
$sSQL .= " CASE WHEN interpolationtype='odd' THEN floor((".$this->fTigerFraction."*(endnumber-startnumber)+startnumber)/2)::int*2+1"; $sSQL .= " CASE WHEN interpolationtype='odd' THEN floor((".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2+1";
$sSQL .= " WHEN interpolationtype='even' THEN ((".$this->fTigerFraction."*(endnumber-startnumber)+startnumber+1)/2)::int*2"; $sSQL .= " WHEN interpolationtype='even' THEN ((".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2";
$sSQL .= " WHEN interpolationtype='all' THEN (".$this->fTigerFraction."*(endnumber-startnumber)+startnumber)::int"; $sSQL .= " WHEN interpolationtype='all' THEN (".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)::int";
$sSQL .= " END as housenumber"; $sSQL .= " END as housenumber";
$sSQL .= " from location_property_tiger where place_id = ".(int)$this->iPlaceID.") as blub1) as blub2"; $sSQL .= " from location_property_tiger where place_id = ".(int)$this->iPlaceID.") as blub1) as blub2";
} }
else if ($this->sType == 'interpolation')
{
$sSQL = "select place_id, partition, 'I' as osm_type, osm_id, 'place' as class, 'house' as type, null admin_level, housenumber, null as street, null as isin, postcode,";
$sSQL .= " calculated_country_code as country_code, parent_place_id, null as linked_place_id, 30 as rank_address, 30 as rank_search,";
$sSQL .= " coalesce(null,0.75-(30::float/40)) as importance, null as indexed_status, null as indexed_date, null as wikipedia, calculated_country_code, ";
$sSQL .= " get_address_by_language(place_id, housenumber, $sLanguagePrefArraySQL) as langaddress,";
$sSQL .= " null as placename,";
$sSQL .= " null as ref,";
if ($this->bExtraTags) $sSQL .= " null as extra,";
if ($this->bNameDetails) $sSQL .= " null as names,";
$sSQL .= " ST_X(point) as lon, ST_Y(point) as lat from (select *, ST_LineInterpolatePoint(linegeo, (housenumber-startnumber::float)/(endnumber-startnumber)::float) as point from ";
$sSQL .= " (select *, ";
$sSQL .= " CASE WHEN interpolationtype='odd' THEN floor((".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2+1";
$sSQL .= " WHEN interpolationtype='even' THEN ((".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)/2)::int*2";
$sSQL .= " WHEN interpolationtype='all' THEN (".$this->fInterpolFraction."*(endnumber-startnumber)+startnumber)::int";
$sSQL .= " END as housenumber";
$sSQL .= " from location_property_osmline where place_id = ".(int)$this->iPlaceID.") as blub1) as blub2";
// testcase: interpolationtype=odd, startnumber=1000, endnumber=1006, fInterpolFraction=1 => housenumber=1007 => error in st_lineinterpolatepoint
// but this will never happen, because if the searched point is that close to the endnumber, the endnumber house will be directly taken from placex (in ReverseGeocode.php line 220)
// and not interpolated
}
else else
{ {
$sSQL = "select placex.place_id, partition, osm_type, osm_id, class, type, admin_level, housenumber, street, isin, postcode, country_code, parent_place_id, linked_place_id, rank_address, rank_search, "; $sSQL = "select placex.place_id, partition, osm_type, osm_id, class, type, admin_level, housenumber, street, isin, postcode, country_code, parent_place_id, linked_place_id, rank_address, rank_search, ";
@@ -172,7 +193,7 @@
if ($this->bAddressDetails) if ($this->bAddressDetails)
{ {
if($this->sType == 'tiger') // to get addressdetails for tiger data, the housenumber is needed if ($this->sType == 'tiger' || $this->sType == 'interpolation') // to get addressdetails for interpolation lines and tiger data, the housenumber is needed
$aAddress = $this->getAddressNames($aPlace['housenumber']); $aAddress = $this->getAddressNames($aPlace['housenumber']);
else else
$aAddress = $this->getAddressNames(); $aAddress = $this->getAddressNames();

114
lib/ReverseGeocode.php
View File

@@ -121,6 +121,7 @@
$fMaxAreaDistance = 1; $fMaxAreaDistance = 1;
$bIsInUnitedStates = false; $bIsInUnitedStates = false;
$bPlaceIsTiger = false; $bPlaceIsTiger = false;
$bPlaceIsLine = false;
while(!$iPlaceID && $fSearchDiam < $fMaxAreaDistance) while(!$iPlaceID && $fSearchDiam < $fMaxAreaDistance)
{ {
$fSearchDiam = $fSearchDiam * 2; $fSearchDiam = $fSearchDiam * 2;
@@ -156,6 +157,106 @@
$iParentPlaceID = $aPlace['parent_place_id']; $iParentPlaceID = $aPlace['parent_place_id'];
$bIsInUnitedStates = ($aPlace['calculated_country_code'] == 'us'); $bIsInUnitedStates = ($aPlace['calculated_country_code'] == 'us');
} }
// if a street or house was found, look in interpolation lines table
if ($iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 || $aPlace['rank_search'] == 30))
{
$fSearchDiam = 0.001;
if ($aPlace['rank_search'] == 30)
{
// if a house was found, the closest road needs to be searched, to use its place id as parent_place_id of the interpolation line
// because a road can be closer to the point than the house from above
$iRoadID = null;
while(!$iRoadID && $fSearchDiam < $fMaxAreaDistance)
{
$fSearchDiam = $fSearchDiam * 2;
$sSQL = 'select place_id ';
$sSQL .= ' FROM placex';
$sSQL .= ' WHERE ST_DWithin('.$sPointSQL.', geometry, '.$fSearchDiam.')';
$sSQL .= ' and (rank_search = 26 or rank_search = 27)';
$sSQL .= ' and class not in (\'waterway\',\'railway\',\'tunnel\',\'bridge\',\'man_made\')';
$sSQL .= ' and indexed_status = 0 ';
$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', geometry) ASC limit 1';
$aPlaceRoad = $this->oDB->getRow($sSQL);
if (PEAR::IsError($aPlace))
{
failInternalError("Could not determine closest place.", $sSQL, $aPlace);
}
$iRoadID = $aPlaceRoad['place_id'];
$iTempPlaceID = $iRoadID;
}
}
else
{
// if a street was found, we can take its place_id as parent_place_id
$iTempPlaceID = $iPlaceID;
}
$sSQL = 'SELECT place_id, parent_place_id, 30 as rank_search, ST_line_locate_point(linegeo,'.$sPointSQL.') as fraction';
//if (CONST_Debug) { $sSQL .= ', housenumber, ST_distance('.$sPointSQL.', centroid) as distance, st_y(centroid) as lat, st_x(centroid) as lon'; }
$sSQL .= ' FROM location_property_osmline WHERE parent_place_id = '.$iTempPlaceID;
$sSQL .= ' AND ST_DWithin('.$sPointSQL.', linegeo, '.$fSearchDiam.') AND indexed_status = 0';
$sSQL .= ' ORDER BY ST_distance('.$sPointSQL.', linegeo) ASC limit 1';
if (CONST_Debug)
{
$sSQL = preg_replace('/limit 1/', 'limit 100', $sSQL);
var_dump($sSQL);
$aAllHouses = $this->oDB->getAll($sSQL);
foreach($aAllHouses as $i)
{
echo $i['housenumber'] . ' | ' . $i['distance'] * 1000 . ' | ' . $i['lat'] . ' | ' . $i['lon']. ' | '. "<br>\n";
}
}
$aPlaceLine = $this->oDB->getRow($sSQL);
if (PEAR::IsError($aPlaceLine))
{
failInternalError("Could not determine closest housenumber on an osm interpolation line.", $sSQL, $aPlaceLine);
}
$iInterpolationLinePlaceID = $aPlaceLine['place_id'];
if ($aPlaceLine)
{
if (CONST_Debug) var_dump('found housenumber in interpolation lines table', $aPlaceLine);
if ($aPlace['rank_search'] == 30)
{
// if a house was already found in placex, we have to find out,
// if the placex house or the interpolated house are closer to the searched point
// distance between point and placex house
$sSQL = 'SELECT ST_distance('.$sPointSQL.', house.geometry) as distance FROM placex as house WHERE house.place_id='.$iPlaceID;
$aDistancePlacex = $this->oDB->getRow($sSQL);
if (PEAR::IsError($aDistancePlacex))
{
failInternalError("Could not determine distance between searched point and placex house.", $sSQL, $aDistancePlacex);
}
$fDistancePlacex = $aDistancePlacex['distance'];
// distance between point and interpolated house (fraction on interpolation line)
$sSQL = 'SELECT ST_distance('.$sPointSQL.', ST_LineInterpolatePoint(linegeo, '.$aPlaceLine['fraction'].')) as distance';
$sSQL .= ' FROM location_property_osmline WHERE place_id = '.$iInterpolationLinePlaceID;
$aDistanceInterpolation = $this->oDB->getRow($sSQL);
if (PEAR::IsError($aDistanceInterpolation))
{
failInternalError("Could not determine distance between searched point and interpolated house.", $sSQL, $aDistanceInterpolation);
}
$fDistanceInterpolation = $aDistanceInterpolation['distance'];
if ($fDistanceInterpolation < $fDistancePlacex)
{
// interpolation is closer to point than placex house
$bPlaceIsLine = true;
$aPlace = $aPlaceLine;
$iPlaceID = $iInterpolationLinePlaceID;
$iParentPlaceID = $aPlaceLine['parent_place_id']; // the street
$fFraction = $aPlaceLine['fraction'];
}
// else: nothing to do, take placex house from above
}
else
{
$bPlaceIsLine = true;
$aPlace = $aPlaceLine;
$iPlaceID = $aPlaceLine['place_id'];
$iParentPlaceID = $aPlaceLine['parent_place_id']; // the street
$fFraction = $aPlaceLine['fraction'];
}
}
}
// Only street found? If it's in the US we can check TIGER data for nearest housenumber // Only street found? If it's in the US we can check TIGER data for nearest housenumber
if ($bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 )) if ($bIsInUnitedStates && $iMaxRank_orig >= 28 && $iPlaceID && ($aPlace['rank_search'] == 26 || $aPlace['rank_search'] == 27 ))
@@ -180,25 +281,25 @@
} }
$aPlaceTiger = $this->oDB->getRow($sSQL); $aPlaceTiger = $this->oDB->getRow($sSQL);
if (PEAR::IsError($aPlace)) if (PEAR::IsError($aPlaceTiger))
{ {
failInternalError("Could not determine closest Tiger place.", $sSQL, $aPlaceTiger); failInternalError("Could not determine closest Tiger place.", $sSQL, $aPlaceTiger);
} }
if ($aPlaceTiger) if ($aPlaceTiger)
{ {
if (CONST_Debug) var_dump('found Tiger place', $aPlaceTiger); if (CONST_Debug) var_dump('found Tiger housenumber', $aPlaceTiger);
$bPlaceIsTiger = true; $bPlaceIsTiger = true;
$aPlace = $aPlaceTiger; $aPlace = $aPlaceTiger;
$iPlaceID = $aPlaceTiger['place_id']; $iPlaceID = $aPlaceTiger['place_id'];
$iParentPlaceID = $aPlaceTiger['parent_place_id']; // the street $iParentPlaceID = $aPlaceTiger['parent_place_id']; // the street
$iFraction = $aPlaceTiger['fraction']; $fFraction = $aPlaceTiger['fraction'];
} }
} }
// The point we found might be too small - use the address to find what it is a child of // The point we found might be too small - use the address to find what it is a child of
if ($iPlaceID && $iMaxRank < 28) if ($iPlaceID && $iMaxRank < 28)
{ {
if ($aPlace['rank_search'] > 28 && $iParentPlaceID && !$bPlaceIsTiger) if (($aPlace['rank_search'] > 28 || $bPlaceIsTiger || $bPlaceIsLine) && $iParentPlaceID)
{ {
$iPlaceID = $iParentPlaceID; $iPlaceID = $iParentPlaceID;
} }
@@ -217,10 +318,9 @@
$iPlaceID = $aPlace['place_id']; $iPlaceID = $aPlace['place_id'];
} }
} }
return array('place_id' => $iPlaceID, return array('place_id' => $iPlaceID,
'type' => $bPlaceIsTiger ? 'tiger' : 'osm', 'type' => $bPlaceIsTiger ? 'tiger' : $bPlaceIsLine ? 'interpolation' : 'osm',
'fraction' => $bPlaceIsTiger ? $iFraction : -1); 'fraction' => ($bPlaceIsTiger || $bPlaceIsLine) ? $fFraction : -1);
} }
} }

3
lib/template/address-json.php
View File

@@ -12,7 +12,8 @@
{ {
if (isset($aPlace['place_id'])) $aFilteredPlaces['place_id'] = $aPlace['place_id']; if (isset($aPlace['place_id'])) $aFilteredPlaces['place_id'] = $aPlace['place_id'];
$aFilteredPlaces['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright"; $aFilteredPlaces['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright";
$sOSMType = ($aPlace['osm_type'] == 'N'?'node':($aPlace['osm_type'] == 'W'?'way':($aPlace['osm_type'] == 'R'?'relation':''))); $sOSMType = ($aPlace['osm_type'] == 'N'?'node':($aPlace['osm_type'] == 'W'?'way':($aPlace['osm_type'] == 'R'?'relation':
($aPlace['osm_type'] == 'T'?'tiger':($aPlace['osm_type'] == 'I'?'interpolation':'')))));
if ($sOSMType) if ($sOSMType)
{ {
$aFilteredPlaces['osm_type'] = $sOSMType; $aFilteredPlaces['osm_type'] = $sOSMType;

3
lib/template/address-jsonv2.php
View File

@@ -12,7 +12,8 @@
{ {
if ($aPlace['place_id']) $aFilteredPlaces['place_id'] = $aPlace['place_id']; if ($aPlace['place_id']) $aFilteredPlaces['place_id'] = $aPlace['place_id'];
$aFilteredPlaces['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright"; $aFilteredPlaces['licence'] = "Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright";
$sOSMType = ($aPlace['osm_type'] == 'N'?'node':($aPlace['osm_type'] == 'W'?'way':($aPlace['osm_type'] == 'R'?'relation':''))); $sOSMType = ($aPlace['osm_type'] == 'N'?'node':($aPlace['osm_type'] == 'W'?'way':($aPlace['osm_type'] == 'R'?'relation':
($aPlace['osm_type'] == 'T'?'tiger':($aPlace['osm_type'] == 'I'?'interpolation':'')))));
if ($sOSMType) if ($sOSMType)
{ {
$aFilteredPlaces['osm_type'] = $sOSMType; $aFilteredPlaces['osm_type'] = $sOSMType;

3
lib/template/address-xml.php
View File

@@ -22,7 +22,8 @@
{ {
echo "<result"; echo "<result";
if ($aPlace['place_id']) echo ' place_id="'.$aPlace['place_id'].'"'; if ($aPlace['place_id']) echo ' place_id="'.$aPlace['place_id'].'"';
$sOSMType = ($aPlace['osm_type'] == 'N'?'node':($aPlace['osm_type'] == 'W'?'way':($aPlace['osm_type'] == 'R'?'relation':''))); $sOSMType = ($aPlace['osm_type'] == 'N'?'node':($aPlace['osm_type'] == 'W'?'way':($aPlace['osm_type'] == 'R'?'relation':
($aPlace['osm_type'] == 'T'?'tiger':($aPlace['osm_type'] == 'I'?'interpolation':'')))));
if ($sOSMType) echo ' osm_type="'.$sOSMType.'"'.' osm_id="'.$aPlace['osm_id'].'"'; if ($sOSMType) echo ' osm_type="'.$sOSMType.'"'.' osm_id="'.$aPlace['osm_id'].'"';
if ($aPlace['ref']) echo ' ref="'.htmlspecialchars($aPlace['ref']).'"'; if ($aPlace['ref']) echo ' ref="'.htmlspecialchars($aPlace['ref']).'"';
if (isset($aPlace['lat'])) echo ' lat="'.htmlspecialchars($aPlace['lat']).'"'; if (isset($aPlace['lat'])) echo ' lat="'.htmlspecialchars($aPlace['lat']).'"';

3
lib/template/search-json.php
View File

@@ -9,7 +9,8 @@
'licence'=>"Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright", 'licence'=>"Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright",
); );
$sOSMType = ($aPointDetails['osm_type'] == 'N'?'node':($aPointDetails['osm_type'] == 'W'?'way':($aPointDetails['osm_type'] == 'R'?'relation':($aPointDetails['osm_type'] == 'T'?'tiger':'')))); $sOSMType = ($aPointDetails['osm_type'] == 'N'?'node':($aPointDetails['osm_type'] == 'W'?'way':($aPointDetails['osm_type'] == 'R'?
'relation':($aPointDetails['osm_type'] == 'T'?'tiger':($aPointDetails['osm_type'] == 'I'?'interpolation':'')))));
if ($sOSMType) if ($sOSMType)
{ {
$aPlace['osm_type'] = $sOSMType; $aPlace['osm_type'] = $sOSMType;

2
lib/template/search-jsonv2.php
View File

@@ -7,7 +7,7 @@
'licence'=>"Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright", 'licence'=>"Data © OpenStreetMap contributors, ODbL 1.0. http://www.openstreetmap.org/copyright",
); );
$sOSMType = ($aPointDetails['osm_type'] == 'N'?'node':($aPointDetails['osm_type'] == 'W'?'way':($aPointDetails['osm_type'] == 'R'?'relation':($aPointDetails['osm_type'] == 'T'?'tiger':'')))); $sOSMType = ($aPointDetails['osm_type'] == 'N'?'node':($aPointDetails['osm_type'] == 'W'?'way':($aPointDetails['osm_type'] == 'R'?'relation':($aPointDetails['osm_type'] == 'T'?'tiger':($aPointDetails['osm_type'] == 'I'?'interpolation':'')))));
if ($sOSMType) if ($sOSMType)
{ {
$aPlace['osm_type'] = $sOSMType; $aPlace['osm_type'] = $sOSMType;

3
lib/template/search-xml.php
View File

@@ -25,7 +25,8 @@
foreach($aSearchResults as $iResNum => $aResult) foreach($aSearchResults as $iResNum => $aResult)
{ {
echo "<place place_id='".$aResult['place_id']."'"; echo "<place place_id='".$aResult['place_id']."'";
$sOSMType = ($aResult['osm_type'] == 'N'?'node':($aResult['osm_type'] == 'W'?'way':($aResult['osm_type'] == 'R'?'relation':($aResult['osm_type'] == 'T'?'tiger':'')))); $sOSMType = ($aPointDetails['osm_type'] == 'N'?'node':($aPointDetails['osm_type'] == 'W'?'way':($aPointDetails['osm_type'] == 'R'?'relation':
($aPointDetails['osm_type'] == 'T'?'tiger':($aPointDetails['osm_type'] == 'I'?'interpolation':'')))));
if ($sOSMType) if ($sOSMType)
{ {
echo " osm_type='$sOSMType'"; echo " osm_type='$sOSMType'";

297
nominatim/index.c
View File

@@ -1,4 +1,6 @@
/* /*
* triggers indexing (reparenting etc.) through setting resetting indexed_status: update placex/osmline set indexed_status = 0 where indexed_status > 0
* triggers placex_update and osmline_update
*/ */
#include <stdio.h> #include <stdio.h>
@@ -71,6 +73,16 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
} }
PQclear(res); PQclear(res);
res = PQprepare(conn, "index_sectors_osmline",
"select geometry_sector,count(*) from location_property_osmline where indexed_status > 0 group by geometry_sector order by geometry_sector",
0, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed preparing index_sectors: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
pg_prepare_params[0] = PG_OID_INT4; pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(conn, "index_nosectors", res = PQprepare(conn, "index_nosectors",
"select 0::integer,count(*) from placex where rank_search = $1 and indexed_status > 0", "select 0::integer,count(*) from placex where rank_search = $1 and indexed_status > 0",
@@ -105,6 +117,27 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
} }
PQclear(res); PQclear(res);
pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(conn, "index_sector_places_osmline",
"select place_id from location_property_osmline where geometry_sector = $1 and indexed_status > 0",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed preparing index_sector_places: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
res = PQprepare(conn, "index_nosector_places_osmline",
"select place_id from location_property_osmline where indexed_status > 0 order by geometry_sector",
0, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed preparing index_nosector_places: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
// Build the data for each thread // Build the data for each thread
thread_data = (struct index_thread_data *)malloc(sizeof(struct index_thread_data)*num_threads); thread_data = (struct index_thread_data *)malloc(sizeof(struct index_thread_data)*num_threads);
for (i = 0; i < num_threads; i++) for (i = 0; i < num_threads; i++)
@@ -127,6 +160,17 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
} }
PQclear(res); PQclear(res);
pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(thread_data[i].conn, "index_osmline",
"update location_property_osmline set indexed_status = 0 where place_id = $1",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed preparing index_osmline: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
/*res = PQexec(thread_data[i].conn, "set enable_seqscan = false"); /*res = PQexec(thread_data[i].conn, "set enable_seqscan = false");
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
@@ -147,8 +191,156 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
fprintf(stderr, "Starting indexing rank (%i to %i) using %i threads\n", rank_min, rank_max, num_threads); fprintf(stderr, "Starting indexing rank (%i to %i) using %i threads\n", rank_min, rank_max, num_threads);
// first for the placex table
for (rank = rank_min; rank <= rank_max; rank++) for (rank = rank_min; rank <= rank_max; rank++)
{ {
// OSMLINE: do reindexing (=> reparenting) for interpolation lines at rank 30, but before all other objects of rank 30
// reason: houses (rank 30) depend on the updated interpolation line, when reparenting (see placex_update in functions.sql)
if (rank == 30)
{
fprintf(stderr, "Starting indexing interpolation lines (location_property_osmline)\n");
rankCountTuples = 0;
rankTotalTuples = 0;
resSectors = PQexecPrepared(conn, "index_sectors_osmline", 0, NULL, 0, NULL, 1);
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
{
fprintf(stderr, "index_sectors_osmline: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resSectors);
exit(EXIT_FAILURE);
}
if (PQftype(resSectors, 0) != PG_OID_INT4)
{
fprintf(stderr, "Sector value has unexpected type\n");
PQclear(resSectors);
exit(EXIT_FAILURE);
}
if (PQftype(resSectors, 1) != PG_OID_INT8)
{
fprintf(stderr, "Sector value has unexpected type\n");
PQclear(resSectors);
exit(EXIT_FAILURE);
}
rankStartTime = time(0);
for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
{
rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)));
}
// do it only if tuples with indexed_status > 0 were found in osmline
int nTuples = PQntuples(resSectors);
if (nTuples > 0)
{
for (iSector = 0; iSector <= nTuples; iSector++)
{
if (iSector > 0)
{
resPlaces = PQgetResult(conn);
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
{
fprintf(stderr, "index_sector_places: SELECT failed: %s\n", PQerrorMessage(conn));
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
if (PQftype(resPlaces, 0) != PG_OID_INT8)
{
fprintf(stderr, "Place_id value has unexpected type\n");
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
resNULL = PQgetResult(conn);
if (resNULL != NULL)
{
fprintf(stderr, "Unexpected non-null response\n");
exit(EXIT_FAILURE);
}
}
if (iSector < nTuples)
{
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
// Get all the place_id's for this sector
paramSector = PGint32(sector);
paramValues[0] = (char *)&paramSector;
paramLengths[0] = sizeof(paramSector);
paramFormats[0] = 1;
if (rankTotalTuples-rankCountTuples < num_threads*1000)
{
// no sectors
iResult = PQsendQueryPrepared(conn, "index_nosector_places_osmline", 0, NULL, 0, NULL, 1);
}
else
{
iResult = PQsendQueryPrepared(conn, "index_sector_places_osmline", 1, paramValues, paramLengths, paramFormats, 1);
}
if (!iResult)
{
fprintf(stderr, "index_sector_places_osmline: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
}
if (iSector > 0)
{
count = 0;
rankPerSecond = 0;
tuples = PQntuples(resPlaces);
if (tuples > 0)
{
// Spawn threads
for (i = 0; i < num_threads; i++)
{
thread_data[i].res = resPlaces;
thread_data[i].tuples = tuples;
thread_data[i].count = &count;
thread_data[i].count_mutex = &count_mutex;
thread_data[i].writer = writer;
thread_data[i].writer_mutex = &writer_mutex;
thread_data[i].table = 0; // use osmline table
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
}
// Monitor threads to give user feedback
sleepcount = 0;
while (count < tuples)
{
usleep(1000);
// Aim for one update per second
if (sleepcount++ > 500)
{
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
fprintf(stderr, " Done %i in %i @ %f per second - Interpolation Lines ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - (rankCountTuples + count)))/(float)rankPerSecond);
sleepcount = 0;
}
}
// Wait for everything to finish
for (i = 0; i < num_threads; i++)
{
pthread_join(thread_data[i].thread, NULL);
}
rankCountTuples += tuples;
}
// Finished sector
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
PQclear(resPlaces);
}
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < nTuples)
{
iSector = nTuples - 1;
}
}
PQclear(resSectors);
}
// Finished rank
fprintf(stderr, "\r Done %i tuples in %i seconds- FINISHED\n", rankCountTuples,(int)(difftime(time(0), rankStartTime)));
if (writer)
{
nominatim_exportXMLEnd(writer);
}
}
fprintf(stderr, "Starting rank %d\n", rank); fprintf(stderr, "Starting rank %d\n", rank);
rankCountTuples = 0; rankCountTuples = 0;
rankPerSecond = 0; rankPerSecond = 0;
@@ -188,6 +380,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
} }
rankStartTime = time(0); rankStartTime = time(0);
for (iSector = 0; iSector <= PQntuples(resSectors); iSector++) for (iSector = 0; iSector <= PQntuples(resSectors); iSector++)
{ {
if (iSector > 0) if (iSector > 0)
@@ -228,13 +421,13 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
paramLengths[1] = sizeof(paramSector); paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1; paramFormats[1] = 1;
if (rankTotalTuples-rankCountTuples < num_threads*1000) if (rankTotalTuples-rankCountTuples < num_threads*1000)
{ {
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
} }
else else
{ {
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1); iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
} }
if (!iResult) if (!iResult)
{ {
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
@@ -260,6 +453,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
thread_data[i].count_mutex = &count_mutex; thread_data[i].count_mutex = &count_mutex;
thread_data[i].writer = writer; thread_data[i].writer = writer;
thread_data[i].writer_mutex = &writer_mutex; thread_data[i].writer_mutex = &writer_mutex;
thread_data[i].table = 1; // use placex table
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]); pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
} }
@@ -304,17 +498,16 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
PQclear(resSectors); PQclear(resSectors);
} }
if (writer)
{
nominatim_exportXMLEnd(writer);
}
// Close all connections if (rank_max == 30)
for (i = 0; i < num_threads; i++)
{ {
PQfinish(thread_data[i].conn); // Close all connections
for (i = 0; i < num_threads; i++)
{
PQfinish(thread_data[i].conn);
}
PQfinish(conn);
} }
PQfinish(conn);
} }
void *nominatim_indexThread(void * thread_data_in) void *nominatim_indexThread(void * thread_data_in)
@@ -324,12 +517,15 @@ void *nominatim_indexThread(void * thread_data_in)
PGresult *res; PGresult *res;
const char *paramValues[1]; const char *paramValues[1];
int paramLengths[1]; int paramLengths[1];
int paramFormats[1]; int paramFormats[1];
uint64_t paramPlaceID; uint64_t paramPlaceID;
uint64_t place_id; uint64_t place_id;
time_t updateStartTime; time_t updateStartTime;
uint table;
table = (uint)(thread_data->table);
while (1) while (1)
{ {
@@ -348,37 +544,58 @@ void *nominatim_indexThread(void * thread_data_in)
if (verbose) fprintf(stderr, " Processing place_id %ld\n", place_id); if (verbose) fprintf(stderr, " Processing place_id %ld\n", place_id);
updateStartTime = time(0); updateStartTime = time(0);
int done = 0; int done = 0;
if (thread_data->writer) if (thread_data->writer)
{ {
nominatim_exportPlaceQueries(place_id, thread_data->conn, &querySet); nominatim_exportPlaceQueries(place_id, thread_data->conn, &querySet);
} }
while(!done) while(!done)
{ {
paramPlaceID = PGint64(place_id); paramPlaceID = PGint64(place_id);
paramValues[0] = (char *)&paramPlaceID; paramValues[0] = (char *)&paramPlaceID;
paramLengths[0] = sizeof(paramPlaceID); paramLengths[0] = sizeof(paramPlaceID);
paramFormats[0] = 1; paramFormats[0] = 1;
res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1); if (table == 1) // table=1 for placex
if (PQresultStatus(res) == PGRES_COMMAND_OK) {
done = 1; res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1);
else }
{ else // table=0 for osmline
if (!strncmp(PQerrorMessage(thread_data->conn), "ERROR: deadlock detected", 25)) {
{ res = PQexecPrepared(thread_data->conn, "index_osmline", 1, paramValues, paramLengths, paramFormats, 1);
fprintf(stderr, "index_placex: UPDATE failed - deadlock, retrying (%ld)\n", place_id); }
PQclear(res); if (PQresultStatus(res) == PGRES_COMMAND_OK)
sleep(rand() % 10); done = 1;
} else
else {
{ if (!strncmp(PQerrorMessage(thread_data->conn), "ERROR: deadlock detected", 25))
fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn)); {
PQclear(res); if (table == 1)
exit(EXIT_FAILURE); {
} fprintf(stderr, "index_placex: UPDATE failed - deadlock, retrying (%ld)\n", place_id);
} }
else
{
fprintf(stderr, "index_osmline: UPDATE failed - deadlock, retrying (%ld)\n", place_id);
}
PQclear(res);
sleep(rand() % 10);
}
else
{
if (table == 1)
{
fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
}
else
{
fprintf(stderr, "index_osmline: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
}
PQclear(res);
exit(EXIT_FAILURE);
}
}
} }
PQclear(res); PQclear(res);
if (difftime(time(0), updateStartTime) > 1) fprintf(stderr, " Slow place_id %ld\n", place_id); if (difftime(time(0), updateStartTime) > 1) fprintf(stderr, " Slow place_id %ld\n", place_id);

1
nominatim/index.h
View File

@@ -14,6 +14,7 @@ struct index_thread_data
pthread_mutex_t * count_mutex; pthread_mutex_t * count_mutex;
xmlTextWriterPtr writer; xmlTextWriterPtr writer;
pthread_mutex_t * writer_mutex; pthread_mutex_t * writer_mutex;
uint table;
}; };
void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile); void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile);
void *nominatim_indexThread(void * thread_data_in); void *nominatim_indexThread(void * thread_data_in);

1696
sql/functions.sql
View File

File diff suppressed because it is too large Load Diff

25
sql/tables.sql
View File

@@ -109,6 +109,27 @@ GRANT SELECT ON location_property_aux TO "{www-user}";
CREATE TABLE location_property_tiger (linegeo GEOMETRY, place_id BIGINT, partition INTEGER, parent_place_id BIGINT, startnumber INTEGER, endnumber INTEGER, interpolationtype TEXT, postcode TEXT); CREATE TABLE location_property_tiger (linegeo GEOMETRY, place_id BIGINT, partition INTEGER, parent_place_id BIGINT, startnumber INTEGER, endnumber INTEGER, interpolationtype TEXT, postcode TEXT);
GRANT SELECT ON location_property_tiger TO "{www-user}"; GRANT SELECT ON location_property_tiger TO "{www-user}";
CREATE TABLE location_property_osmline (
linegeo GEOMETRY,
place_id BIGINT NOT NULL,
partition INTEGER,
osm_id BIGINT,
parent_place_id BIGINT,
startnumber INTEGER,
endnumber INTEGER,
interpolationtype TEXT,
admin_level INTEGER,
street TEXT,
postcode TEXT,
calculated_country_code VARCHAR(2),
geometry_sector INTEGER,
indexed_status INTEGER,
indexed_date TIMESTAMP);
CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline (place_id) {ts:search-index};
CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline (parent_place_id) {ts:search-index};
GRANT SELECT ON location_property_osmline TO "{www-user}";
drop table IF EXISTS search_name; drop table IF EXISTS search_name;
CREATE TABLE search_name ( CREATE TABLE search_name (
place_id BIGINT, place_id BIGINT,
@@ -167,13 +188,15 @@ GRANT SELECT ON planet_osm_ways to "{www-user}" ;
GRANT SELECT ON planet_osm_rels to "{www-user}" ; GRANT SELECT ON planet_osm_rels to "{www-user}" ;
GRANT SELECT on location_area to "{www-user}" ; GRANT SELECT on location_area to "{www-user}" ;
-- insert creates the location tagbles, creates location indexes if indexed == true -- insert creates the location tables, creates location indexes if indexed == true
CREATE TRIGGER placex_before_insert BEFORE INSERT ON placex CREATE TRIGGER placex_before_insert BEFORE INSERT ON placex
FOR EACH ROW EXECUTE PROCEDURE placex_insert(); FOR EACH ROW EXECUTE PROCEDURE placex_insert();
-- update insert creates the location tables -- update insert creates the location tables
CREATE TRIGGER placex_before_update BEFORE UPDATE ON placex CREATE TRIGGER placex_before_update BEFORE UPDATE ON placex
FOR EACH ROW EXECUTE PROCEDURE placex_update(); FOR EACH ROW EXECUTE PROCEDURE placex_update();
CREATE TRIGGER osmline_before_update BEFORE UPDATE ON location_property_osmline
FOR EACH ROW EXECUTE PROCEDURE osmline_update();
-- diff update triggers -- diff update triggers
CREATE TRIGGER placex_before_delete AFTER DELETE ON placex CREATE TRIGGER placex_before_delete AFTER DELETE ON placex

308
tests/features/db/import/interpolation.feature
View File

@@ -2,41 +2,23 @@
Feature: Import of address interpolations Feature: Import of address interpolations
Tests that interpolated addresses are added correctly Tests that interpolated addresses are added correctly
Scenario: Simple even two point interpolation Scenario: Simple even interpolation line with two points
Given the place nodes Given the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | osm_type | class | type | housenumber | geometry
| 1 | place | house | 2 | 1 1 | 1 | N | place | house | 2 | 1 1
| 2 | place | house | 6 | 1 1.001 | 2 | N | place | house | 6 | 1 1.001
And the place ways And the place ways
| osm_id | class | type | housenumber | geometry | osm_id | osm_type | class | type | housenumber | geometry
| 1 | place | houses | even | 1 1, 1 1.001 | 1 | W | place | houses | even | 1 1, 1 1.001
And the ways And the ways
| id | nodes | id | nodes
| 1 | 1,2 | 1 | 1,2
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 1,1.0005 | 2 | 6 | 1 1, 1 1.001
Scenario: Simple even two point interpolation with zero beginning Scenario: Backwards even two point interpolation line
Given the place nodes
| osm_id | class | type | housenumber | geometry
| 1 | place | house | 0 | 1 1
| 2 | place | house | 8 | 1 1.001
And the place ways
| osm_id | class | type | housenumber | geometry
| 1 | place | houses | even | 1 1, 1 1.001
And the ways
| id | nodes
| 1 | 1,2
When importing
Then way 1 expands to housenumbers
| housenumber | centroid
| 2 | 1,1.00025
| 4 | 1,1.0005
| 6 | 1,1.00075
Scenario: Backwards even two point interpolation
Given the place nodes Given the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | 1 1 | 1 | place | house | 2 | 1 1
@@ -48,77 +30,9 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 2,1 | 1 | 2,1
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 1,1.0005 | 2 | 6 | 1 1, 1 1.001
Scenario: Even two point interpolation with odd beginning
Given the place nodes
| osm_id | class | type | housenumber | geometry
| 1 | place | house | 11 | 1 1
| 2 | place | house | 16 | 1 1.001
And the place ways
| osm_id | class | type | housenumber | geometry
| 1 | place | houses | even | 1 1, 1 1.001
And the ways
| id | nodes
| 1 | 1,2
When importing
Then way 1 expands to housenumbers
| housenumber | centroid
| 12 | 1,1.0002
| 14 | 1,1.0006
Scenario: Even two point interpolation with odd end
Given the place nodes
| osm_id | class | type | housenumber | geometry
| 1 | place | house | 10 | 1 1
| 2 | place | house | 15 | 1 1.001
And the place ways
| osm_id | class | type | housenumber | geometry
| 1 | place | houses | even | 1 1, 1 1.001
And the ways
| id | nodes
| 1 | 1,2
When importing
Then way 1 expands to housenumbers
| housenumber | centroid
| 12 | 1,1.0004
| 14 | 1,1.0008
Scenario: Reverse even two point interpolation with odd beginning
Given the place nodes
| osm_id | class | type | housenumber | geometry
| 1 | place | house | 11 | 1 1
| 2 | place | house | 16 | 1 1.001
And the place ways
| osm_id | class | type | housenumber | geometry
| 1 | place | houses | even | 1 1.001, 1 1
And the ways
| id | nodes
| 1 | 2,1
When importing
Then way 1 expands to housenumbers
| housenumber | centroid
| 12 | 1,1.0002
| 14 | 1,1.0006
Scenario: Reverse even two point interpolation with odd end
Given the place nodes
| osm_id | class | type | housenumber | geometry
| 1 | place | house | 10 | 1 1
| 2 | place | house | 15 | 1 1.001
And the place ways
| osm_id | class | type | housenumber | geometry
| 1 | place | houses | even | 1 1.001, 1 1
And the ways
| id | nodes
| 1 | 2,1
When importing
Then way 1 expands to housenumbers
| housenumber | centroid
| 12 | 1,1.0004
| 14 | 1,1.0008
Scenario: Simple odd two point interpolation Scenario: Simple odd two point interpolation
Given the place nodes Given the place nodes
@@ -132,29 +46,9 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,2 | 1 | 1,2
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 3 | 1,1.0002 | 1 | 11 | 1 1, 1 1.001
| 5 | 1,1.0004
| 7 | 1,1.0006
| 9 | 1,1.0008
Scenario: Odd two point interpolation with even beginning
Given the place nodes
| osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | 1 1
| 2 | place | house | 7 | 1 1.001
And the place ways
| osm_id | class | type | housenumber | geometry
| 1 | place | houses | odd | 1 1, 1 1.001
And the ways
| id | nodes
| 1 | 1,2
When importing
Then way 1 expands to housenumbers
| housenumber | centroid
| 3 | 1,1.0002
| 5 | 1,1.0006
Scenario: Simple all two point interpolation Scenario: Simple all two point interpolation
Given the place nodes Given the place nodes
@@ -168,27 +62,11 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,2 | 1 | 1,2
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 2 | 1,1.0005 | 1 | 3 | 1 1, 1 1.001
Scenario: Simple numbered two point interpolation Scenario: Even two point interpolation line with intermediate empty node
Given the place nodes
| osm_id | class | type | housenumber | geometry
| 1 | place | house | 3 | 1 1
| 2 | place | house | 9 | 1 1.001
And the place ways
| osm_id | class | type | housenumber | geometry
| 1 | place | houses | 3 | 1 1, 1 1.001
And the ways
| id | nodes
| 1 | 1,2
When importing
Then way 1 expands to housenumbers
| housenumber | centroid
| 6 | 1,1.0005
Scenario: Even two point interpolation with intermediate empty node
Given the place nodes Given the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | 1 1 | 1 | place | house | 2 | 1 1
@@ -200,14 +78,11 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,3,2 | 1 | 1,3,2
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 1,1.0005 | 2 | 10 | 1 1, 1 1.001, 1.001 1.001
| 6 | 1,1.001
| 8 | 1.0005,1.001
Scenario: Even two point interpolation line with intermediate duplicated empty node
Scenario: Even two point interpolation with intermediate duplicated empty node
Given the place nodes Given the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | 1 1 | 1 | place | house | 2 | 1 1
@@ -219,13 +94,11 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,3,3,2 | 1 | 1,3,3,2
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 1,1.0005 | 2 | 10 | 1 1, 1 1.001, 1.001 1.001
| 6 | 1,1.001
| 8 | 1.0005,1.001
Scenario: Simple even three point interpolation Scenario: Simple even three point interpolation line
Given the place nodes Given the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | 1 1 | 1 | place | house | 2 | 1 1
@@ -238,14 +111,12 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,3,2 | 1 | 1,3,2
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 1,1.00025 | 2 | 10 | 1 1, 1 1.001
| 6 | 1,1.0005 | 10 | 14 | 1 1.001, 1.001 1.001
| 8 | 1,1.00075
| 12 | 1.0005,1.001
Scenario: Simple even four point interpolation Scenario: Simple even four point interpolation line
Given the place nodes Given the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | 1 1 | 1 | place | house | 2 | 1 1
@@ -259,15 +130,13 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,3,2,4 | 1 | 1,3,2,4
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 1,1.00025 | 2 | 10 | 1 1, 1 1.001
| 6 | 1,1.0005 | 10 | 14 | 1 1.001, 1.001 1.001
| 8 | 1,1.00075 | 14 | 18 | 1.001 1.001, 1.001 1.002
| 12 | 1.0005,1.001
| 16 | 1.001,1.0015
Scenario: Reverse simple even three point interpolation Scenario: Reverse simple even three point interpolation line
Given the place nodes Given the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | 1 1 | 1 | place | house | 2 | 1 1
@@ -280,14 +149,12 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 2,3,1 | 1 | 2,3,1
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 1,1.00025 | 2 | 10 | 1 1, 1 1.001
| 6 | 1,1.0005 | 10 | 14 | 1 1.001, 1.001 1.001
| 8 | 1,1.00075
| 12 | 1.0005,1.001
Scenario: Even three point interpolation with odd center point Scenario: Even three point interpolation line with odd center point
Given the place nodes Given the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | 1 1 | 1 | place | house | 2 | 1 1
@@ -300,12 +167,12 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,3,2 | 1 | 1,3,2
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 1,1.0004 | 2 | 7 | 1 1, 1 1.001
| 6 | 1,1.0008 | 7 | 8 | 1 1.001, 1.001 1.001
Scenario: Interpolation on self-intersecting way Scenario: Interpolation line with self-intersecting way
Given the place nodes Given the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | 0 0 | 1 | place | house | 2 | 0 0
@@ -318,12 +185,12 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,2,3,2 | 1 | 1,2,3,2
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 0,0.0005 | 2 | 6 | 0 0, 0 0.001
| 8 | 0,0.0015 | 6 | 10 | 0 0.001, 0 0.002
Scenario: Interpolation on self-intersecting way II Scenario: Interpolation line with self-intersecting way II
Given the place nodes Given the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | 0 0 | 1 | place | house | 2 | 0 0
@@ -335,24 +202,24 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,2,3,2 | 1 | 1,2,3,2
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 0,0.0005 | 2 | 6 | 0 0, 0 0.001
Scenario: addr:street on interpolation way Scenario: addr:street on interpolation way
Given the scene parallel-road Given the scene parallel-road
And the place nodes And the place nodes
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 1 | place | house | 2 | :n-middle-w | 1 | place | house | 2 | :n-middle-w
| 2 | place | house | 6 | :n-middle-e | 2 | place | house | 6 | :n-middle-e
| 3 | place | house | 12 | :n-middle-w | 3 | place | house | 12 | :n-middle-w
| 4 | place | house | 16 | :n-middle-e | 4 | place | house | 16 | :n-middle-e
And the place ways And the place ways
| osm_id | class | type | housenumber | street | geometry | osm_id | class | type | housenumber | street | geometry
| 10 | place | houses | even | | :w-middle | 10 | place | houses | even | | :w-middle
| 11 | place | houses | even | Cloud Street | :w-middle | 11 | place | houses | even | Cloud Street | :w-middle
And the place ways And the place ways
| osm_id | class | type | name | geometry | osm_id | class | type | name | geometry
| 2 | highway | tertiary | 'name' : 'Sun Way' | :w-north | 2 | highway | tertiary | 'name' : 'Sun Way' | :w-north
| 3 | highway | tertiary | 'name' : 'Cloud Street' | :w-south | 3 | highway | tertiary | 'name' : 'Cloud Street' | :w-south
@@ -360,49 +227,49 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 10 | 1,100,101,102,2 | 10 | 1,100,101,102,2
| 11 | 3,200,201,202,4 | 11 | 3,200,201,202,4
When importing When importing
Then table placex contains Then table placex contains
| object | parent_place_id | object | parent_place_id
| N1 | W2 | N1 | W2
| N2 | W2 | N2 | W2
| N3 | W3 | N3 | W3
| N4 | W3 | N4 | W3
| W10 | W2 Then table location_property_osmline contains
| W11 | W3 | object | parent_place_id | startnumber | endnumber
And way 10 expands exactly to housenumbers 4 | W10 | W2 | 2 | 6
And way 11 expands exactly to housenumbers 14 | W11 | W3 | 12 | 16
Scenario: addr:street on housenumber way Scenario: addr:street on housenumber way
Given the scene parallel-road Given the scene parallel-road
And the place nodes And the place nodes
| osm_id | class | type | housenumber | street | geometry | osm_id | class | type | housenumber | street | geometry
| 1 | place | house | 2 | | :n-middle-w | 1 | place | house | 2 | | :n-middle-w
| 2 | place | house | 6 | | :n-middle-e | 2 | place | house | 6 | | :n-middle-e
| 3 | place | house | 12 | Cloud Street | :n-middle-w | 3 | place | house | 12 | Cloud Street | :n-middle-w
| 4 | place | house | 16 | Cloud Street | :n-middle-e | 4 | place | house | 16 | Cloud Street | :n-middle-e
And the place ways And the place ways
| osm_id | class | type | housenumber | geometry | osm_id | class | type | housenumber | geometry
| 10 | place | houses | even | :w-middle | 10 | place | houses | even | :w-middle
| 11 | place | houses | even | :w-middle | 11 | place | houses | even | :w-middle
And the place ways And the place ways
| osm_id | class | type | name | geometry | osm_id | class | type | name | geometry
| 2 | highway | tertiary | 'name' : 'Sun Way' | :w-north | 2 | highway | tertiary | 'name' : 'Sun Way' | :w-north
| 3 | highway | tertiary | 'name' : 'Cloud Street' | :w-south | 3 | highway | tertiary | 'name' : 'Cloud Street' | :w-south
And the ways And the ways
| id | nodes | id | nodes
| 10 | 1,100,101,102,2 | 10 | 1,100,101,102,2
| 11 | 3,200,201,202,4 | 11 | 3,200,201,202,4
When importing When importing
Then table placex contains Then table placex contains
| object | parent_place_id | object | parent_place_id
| N1 | W2 | N1 | W2
| N2 | W2 | N2 | W2
| N3 | W3 | N3 | W3
| N4 | W3 | N4 | W3
| W10 | W2 And table location_property_osmline contains
| W11 | W3 | object | parent_place_id | startnumber | endnumber
And way 10 expands exactly to housenumbers 4 | W10 | W2 | 2 | 6
And way 11 expands exactly to housenumbers 14 | W11 | W3 | 12 | 16
Scenario: Geometry of points and way don't match (github #253) Scenario: Geometry of points and way don't match (github #253)
Given the place nodes Given the place nodes
@@ -417,10 +284,10 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,2,3 | 1 | 1,2,3
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 4 | 144.963016723312,-37.7629464422819+-0.000005 | 2 | 6 | 144.9629794 -37.7630755, 144.9630541 -37.7628174
| 8 | 144.9631440856,-37.762223694978+-0.000005 | 6 | 10 | 144.9630541 -37.7628174, 144.9632341 -37.76163
Scenario: Place with missing address information Scenario: Place with missing address information
Given the place nodes Given the place nodes
@@ -435,7 +302,6 @@ Feature: Import of address interpolations
| id | nodes | id | nodes
| 1 | 1,2,3 | 1 | 1,2,3
When importing When importing
Then way 1 expands to housenumbers Then way 1 expands to lines
| housenumber | centroid | startnumber | endnumber | geometry
| 25 | 0.0001,0.0002 | 23 | 29 | 0.0001 0.0001, 0.0001 0.0002, 0.0001 0.0004
| 27 | 0.0001,0.0003

55
tests/features/db/update/interpolation.feature
View File

@@ -23,17 +23,19 @@ Feature: Update of address interpolations
| object | parent_place_id | object | parent_place_id
| N1 | W2 | N1 | W2
| N2 | W2 | N2 | W2
| W10 | W2 And table location_property_osmline contains
And way 10 expands exactly to housenumbers 4 | object | parent_place_id | startnumber | endnumber
| W10 | W2 | 2 | 6
When updating place ways When updating place ways
| osm_id | class | type | housenumber | street | geometry | osm_id | class | type | housenumber | street | geometry
| 10 | place | houses | even | Cloud Street | :w-middle | 10 | place | houses | even | Cloud Street | :w-middle
Then way 10 expands exactly to housenumbers 4 Then table placex contains
And table placex contains
| object | parent_place_id | object | parent_place_id
| N1 | W3 | N1 | W3
| N2 | W3 | N2 | W3
| W10 | W3 And table location_property_osmline contains
| object | parent_place_id | startnumber | endnumber
| W10 | W3 | 2 | 6
@Fail @Fail
Scenario: addr:street added to housenumbers Scenario: addr:street added to housenumbers
@@ -57,18 +59,20 @@ Feature: Update of address interpolations
| object | parent_place_id | object | parent_place_id
| N1 | W2 | N1 | W2
| N2 | W2 | N2 | W2
| W10 | W2 And table location_property_osmline contains
And way 10 expands exactly to housenumbers 4 | object | parent_place_id | startnumber | endnumber
| W10 | W2 | 2 | 6
When updating place nodes When updating place nodes
| osm_id | class | type | street | housenumber | geometry | osm_id | class | type | street | housenumber | geometry
| 1 | place | house | Cloud Street| 2 | :n-middle-w | 1 | place | house | Cloud Street| 2 | :n-middle-w
| 2 | place | house | Cloud Street| 6 | :n-middle-e | 2 | place | house | Cloud Street| 6 | :n-middle-e
Then way 10 expands exactly to housenumbers 4 Then table placex contains
And table placex contains
| object | parent_place_id | object | parent_place_id
| N1 | W3 | N1 | W3
| N2 | W3 | N2 | W3
| W10 | W3 And table location_property_osmline contains
| object | parent_place_id | startnumber | endnumber
| W10 | W3 | 2 | 6
Scenario: interpolation tag removed Scenario: interpolation tag removed
@@ -92,10 +96,11 @@ Feature: Update of address interpolations
| object | parent_place_id | object | parent_place_id
| N1 | W2 | N1 | W2
| N2 | W2 | N2 | W2
| W10 | W2 And table location_property_osmline contains
And way 10 expands exactly to housenumbers 4 | object | parent_place_id | startnumber | endnumber
| W10 | W2 | 2 | 6
When marking for delete W10 When marking for delete W10
Then way 10 expands to no housenumbers Then table location_property_osmline has no entry for W10
And table placex contains And table placex contains
| object | parent_place_id | object | parent_place_id
| N1 | W2 | N1 | W2
@@ -122,17 +127,19 @@ Feature: Update of address interpolations
| object | parent_place_id | object | parent_place_id
| N1 | W2 | N1 | W2
| N2 | W2 | N2 | W2
| W10 | W2 And table location_property_osmline contains
And way 10 expands exactly to housenumbers 4 | object | parent_place_id | startnumber | endnumber
| W10 | W2 | 2 | 6
When updating place ways When updating place ways
| osm_id | class | type | name | geometry | osm_id | class | type | name | geometry
| 3 | highway | unclassified | 'name' : 'Cloud Street' | :w-south | 3 | highway | unclassified | 'name' : 'Cloud Street' | :w-south
Then way 10 expands exactly to housenumbers 4 Then table placex contains
And table placex contains
| object | parent_place_id | object | parent_place_id
| N1 | W3 | N1 | W3
| N2 | W3 | N2 | W3
| W10 | W3 And table location_property_osmline contains
| object | parent_place_id | startnumber | endnumber
| W10 | W3 | 2 | 6
Scenario: referenced road deleted Scenario: referenced road deleted
@@ -156,12 +163,14 @@ Feature: Update of address interpolations
| object | parent_place_id | object | parent_place_id
| N1 | W3 | N1 | W3
| N2 | W3 | N2 | W3
| W10 | W3 And table location_property_osmline contains
And way 10 expands exactly to housenumbers 4 | object | parent_place_id | startnumber | endnumber
| W10 | W3 | 2 | 6
When marking for delete W3 When marking for delete W3
Then way 10 expands exactly to housenumbers 4 Then table placex contains
And table placex contains
| object | parent_place_id | object | parent_place_id
| N1 | W2 | N1 | W2
| N2 | W2 | N2 | W2
| W10 | W2 And table location_property_osmline contains
| object | parent_place_id | startnumber | endnumber
| W10 | W2 | 2 | 6

54
tests/steps/db_results.py
View File

@@ -35,6 +35,7 @@ def check_placex_names(step, osmtyp, osmid):
@step(u'table ([a-z_]+) contains$') @step(u'table ([a-z_]+) contains$')
def check_placex_content(step, tablename): def check_placex_content(step, tablename):
""" check that the given lines are in the given table """ check that the given lines are in the given table
@@ -49,13 +50,21 @@ def check_placex_content(step, tablename):
q = 'SELECT *' q = 'SELECT *'
if tablename == 'placex': if tablename == 'placex':
q = q + ", ST_X(centroid) as clat, ST_Y(centroid) as clon" q = q + ", ST_X(centroid) as clat, ST_Y(centroid) as clon"
q = q + ", ST_GeometryType(geometry) as geometrytype" if tablename == 'location_property_osmline':
q = q + ' FROM %s where osm_type = %%s and osm_id = %%s' % (tablename,) q = q + ' FROM %s where osm_id = %%s' % (tablename,)
else:
q = q + ' FROM %s where osm_type = %%s and osm_id = %%s' % (tablename,)
if cls is None: if cls is None:
params = (osmtype, osmid) if tablename == 'location_property_osmline':
params = (osmid,)
else:
params = (osmtype, osmid)
else: else:
q = q + ' and class = %s' q = q + ' and class = %s'
params = (osmtype, osmid, cls) if tablename == 'location_property_osmline':
params = (osmid, cls)
else:
params = (osmtype, osmid, cls)
cur.execute(q, params) cur.execute(q, params)
assert(cur.rowcount > 0) assert(cur.rowcount > 0)
for res in cur: for res in cur:
@@ -92,6 +101,18 @@ def check_placex_missing(step, tablename, osmtyp, osmid, placeclass):
cur.close() cur.close()
world.conn.commit() world.conn.commit()
@step(u'table location_property_osmline has no entry for W(\d+)?')
def check_osmline_missing(step, osmid):
cur = world.conn.cursor()
try:
q = 'SELECT count(*) FROM location_property_osmline where osm_id = %s' % (osmid, )
cur.execute(q)
numres = cur.fetchone()[0]
assert_equals (numres, 0)
finally:
cur.close()
world.conn.commit()
@step(u'search_name table contains$') @step(u'search_name table contains$')
def check_search_name_content(step): def check_search_name_content(step):
cur = world.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) cur = world.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
@@ -134,12 +155,35 @@ def check_interpolated_housenumbers(step, nodeid):
FROM placex WHERE osm_type = 'W' and osm_id = %s FROM placex WHERE osm_type = 'W' and osm_id = %s
and class = 'place' and type = 'address'""", and class = 'place' and type = 'address'""",
(int(nodeid),)) (int(nodeid),))
assert_equals(len(numbers), cur.rowcount) assert_equals(len(numbers), cur.rowcount)
for r in cur: for r in cur:
assert_in(r["housenumber"], numbers) assert_in(r["housenumber"], numbers)
world.match_geometry((r['clat'], r['clon']), numbers[r["housenumber"]]) world.match_geometry((r['clat'], r['clon']), numbers[r["housenumber"]])
del numbers[r["housenumber"]] del numbers[r["housenumber"]]
@step(u'way (\d+) expands to lines')
def check_interpolation_lines(step, wayid):
"""Check that the correct interpolation line has been entered in
location_property_osmline for the given source line/nodes. Expected are three columns:
startnumber, endnumber and linegeo
"""
lines = {}
for line in step.hashes:
assert line["startnumber"] not in lines
lines[line["startnumber"]] = {'endnumber': line["endnumber"], 'geometry': line["geometry"]}
cur = world.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
cur.execute("""SELECT startnumber::text, endnumber::text, st_astext(linegeo) as geometry
FROM location_property_osmline WHERE osm_id = %s""",
(int(wayid),))
assert_equals(len(lines), cur.rowcount)
for r in cur:
assert_in(r["startnumber"], lines)
assert_equals(r["endnumber"], lines[r["startnumber"]]["endnumber"])
linegeo = str(str(r["geometry"].split('(')[1]).split(')')[0]).replace(',', ', ')
assert_equals(linegeo, lines[r["startnumber"]]["geometry"])
del lines[r["startnumber"]]
@step(u'way (\d+) expands exactly to housenumbers ([0-9,]*)') @step(u'way (\d+) expands exactly to housenumbers ([0-9,]*)')
def check_interpolated_housenumber_list(step, nodeid, numberlist): def check_interpolated_housenumber_list(step, nodeid, numberlist):
""" Checks that the interpolated house numbers corresponds """ Checks that the interpolated house numbers corresponds
@@ -149,7 +193,7 @@ def check_interpolated_housenumber_list(step, nodeid, numberlist):
cur = world.conn.cursor() cur = world.conn.cursor()
cur.execute("""SELECT housenumber FROM placex cur.execute("""SELECT housenumber FROM placex
WHERE osm_type = 'W' and osm_id = %s WHERE osm_type = 'W' and osm_id = %s
and class = 'place' and type = 'address'""", (int(nodeid),)) and class = 'place' and type = 'address'""", (int(nodeid),))
for r in cur: for r in cur:
assert_in(r[0], expected, "Unexpected house number %s for node %s." % (r[0], nodeid)) assert_in(r[0], expected, "Unexpected house number %s for node %s." % (r[0], nodeid))
expected.remove(r[0]) expected.remove(r[0])

14
tests/steps/db_setup.py
View File

@@ -23,6 +23,7 @@ import os
import subprocess import subprocess
import random import random
import base64 import base64
import sys
psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
@@ -143,8 +144,8 @@ def import_set_scene(step, scene):
@step(u'the (named )?place (node|way|area)s') @step(u'the (named )?place (node|way|area)s')
def import_place_table_nodes(step, named, osmtype): def import_place_table_nodes(step, named, osmtype):
"""Insert a list of nodes into the placex table. """Insert a list of nodes into the place table.
Expects a table where columns are named in the same way as placex. Expects a table where columns are named in the same way as place.
""" """
cur = world.conn.cursor() cur = world.conn.cursor()
cur.execute('ALTER TABLE place DISABLE TRIGGER place_before_insert') cur.execute('ALTER TABLE place DISABLE TRIGGER place_before_insert')
@@ -214,18 +215,19 @@ def import_database(step):
""" Runs the actual indexing. """ """ Runs the actual indexing. """
world.run_nominatim_script('setup', 'create-functions', 'create-partition-functions') world.run_nominatim_script('setup', 'create-functions', 'create-partition-functions')
cur = world.conn.cursor() cur = world.conn.cursor()
#world.db_dump_table('place')
cur.execute("""insert into placex (osm_type, osm_id, class, type, name, admin_level, cur.execute("""insert into placex (osm_type, osm_id, class, type, name, admin_level,
housenumber, street, addr_place, isin, postcode, country_code, extratags, housenumber, street, addr_place, isin, postcode, country_code, extratags,
geometry) select * from place""") geometry) select * from place where not (class='place' and type='houses' and osm_type='W')""")
cur.execute("""select insert_osmline (osm_id, housenumber, street, addr_place, postcode, country_code, geometry) from place where class='place' and type='houses' and osm_type='W'""")
world.conn.commit() world.conn.commit()
world.run_nominatim_script('setup', 'index', 'index-noanalyse') world.run_nominatim_script('setup', 'index', 'index-noanalyse')
#world.db_dump_table('placex') #world.db_dump_table('placex')
#world.db_dump_table('location_property_osmline')
@step(u'updating place (node|way|area)s') @step(u'updating place (node|way|area)s')
def update_place_table_nodes(step, osmtype): def update_place_table_nodes(step, osmtype):
""" Replace a geometry in place by reinsertion and reindex database. """ Replace a geometry in place by reinsertion and reindex database."""
"""
world.run_nominatim_script('setup', 'create-functions', 'create-partition-functions', 'enable-diff-updates') world.run_nominatim_script('setup', 'create-functions', 'create-partition-functions', 'enable-diff-updates')
if osmtype == 'node': if osmtype == 'node':
_insert_place_table_nodes(step.hashes, False) _insert_place_table_nodes(step.hashes, False)

7
tests/steps/terrain.py
View File

@@ -123,16 +123,19 @@ def match_geometry(coord, matchstring):
logger.debug("Distances expected: %f, got: %f" % (expdist, dist)) logger.debug("Distances expected: %f, got: %f" % (expdist, dist))
assert dist <= expdist, "Geometry too far away, expected: %f, got: %f" % (expdist, dist) assert dist <= expdist, "Geometry too far away, expected: %f, got: %f" % (expdist, dist)
@world.absorb
def print_statement(element):
print '\n\n\n'+str(element)+'\n\n\n'
@world.absorb @world.absorb
def db_dump_table(table): def db_dump_table(table):
cur = world.conn.cursor() cur = world.conn.cursor()
cur.execute('SELECT * FROM %s' % table) cur.execute('SELECT * FROM %s' % table)
print '<<<<<<< BEGIN OF TABLE DUMP %s' % table print '\n\n\n<<<<<<< BEGIN OF TABLE DUMP %s' % table
for res in cur: for res in cur:
print res print res
print '<<<<<<< END OF TABLE DUMP %s' % table print '<<<<<<< END OF TABLE DUMP %s\n\n\n' % table
@world.absorb @world.absorb
def db_drop_database(name): def db_drop_database(name):

72
utils/setup.php
View File

@@ -404,6 +404,8 @@
echo '.'; echo '.';
if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection)); if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection));
echo '.'; echo '.';
if (!pg_query($oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($oDB->connection));
echo '.';
if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection)); if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection));
echo '.'; echo '.';
if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection)); if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection));
@@ -445,27 +447,70 @@
echo "Load Data\n"; echo "Load Data\n";
$aDBInstances = array(); $aDBInstances = array();
$aQueriesPlacex = array();
$aQueriesOsmline = array();
// the query is divided into parcels, so that the work between the processes, i.e. the DBInstances, will be evenly distributed
$iNumberOfParcels = 100;
for($i = 0; $i < $iNumberOfParcels; $i++)
{
$sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
$sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
$sSQL .= 'geometry) select * from place where osm_id % '.$iNumberOfParcels.' = '.$i.' and not ';
$sSQL .= '(class=\'place\' and type=\'houses\' and osm_type=\'W\' and ST_GeometryType(geometry) = \'ST_LineString\');';
array_push($aQueriesPlacex, $sSQL);
$sSQL = 'select insert_osmline (osm_id, housenumber, street, addr_place, postcode, country_code, ';
$sSQL .= 'geometry) from place where osm_id % '.$iNumberOfParcels.' = '.$i.' and ';
$sSQL .= 'class=\'place\' and type=\'houses\' and osm_type=\'W\' and ST_GeometryType(geometry) = \'ST_LineString\'';
array_push($aQueriesOsmline, $sSQL);
}
for($i = 0; $i < $iInstances; $i++) for($i = 0; $i < $iInstances; $i++)
{ {
$aDBInstances[$i] =& getDB(true); $aDBInstances[$i] =& getDB(true);
$sSQL = 'insert into placex (osm_type, osm_id, class, type, name, admin_level, ';
$sSQL .= 'housenumber, street, addr_place, isin, postcode, country_code, extratags, ';
$sSQL .= 'geometry) select * from place where osm_id % '.$iInstances.' = '.$i;
if ($aCMDResult['verbose']) echo "$sSQL\n";
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection));
} }
$bAnyBusy = true; // now execute the query blocks, in the first round for placex, then for osmline,
while($bAnyBusy) // because insert_osmline depends on the placex table
echo 'Inserting from place to placex.';
$aQueries = $aQueriesPlacex;
for($j = 0; $j < 2; $j++)
{ {
$bAnyBusy = false; $bAnyBusy = true;
for($i = 0; $i < $iInstances; $i++) while($bAnyBusy)
{ {
if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; $bAnyBusy = false;
for($i = 0; $i < $iInstances; $i++)
{
if (pg_connection_busy($aDBInstances[$i]->connection))
{
$bAnyBusy = true;
}
else if (count($aQueries) > 0)
{
$query = array_pop($aQueries);
if (!pg_send_query($aDBInstances[$i]->connection, $query))
{
fail(pg_last_error($oDB->connection));
}
else
{
pg_get_result($aDBInstances[$i]->connection);
$bAnyBusy = true;
}
}
}
sleep(1);
echo '.';
}
echo "\n";
if ($j == 0) //for the second round with osmline
{
echo 'Inserting from place to osmline.';
$aQueries = $aQueriesOsmline;
} }
sleep(1);
echo '.';
} }
echo "\n";
echo "Reanalysing database...\n"; echo "Reanalysing database...\n";
pgsqlRunScript('ANALYSE'); pgsqlRunScript('ANALYSE');
} }
@@ -552,7 +597,6 @@
$sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y "; $sSQL .= "avg(st_x(st_centroid(geometry))) as x,avg(st_y(st_centroid(geometry))) as y ";
$sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x"; $sSQL .= "from placex where postcode is not null group by calculated_country_code,postcode) as x";
if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
$sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) "; $sSQL = "insert into placex (osm_type,osm_id,class,type,postcode,calculated_country_code,geometry) ";
$sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',"; $sSQL .= "select 'P',nextval('seq_postcodes'),'place','postcode',postcode,'us',";
$sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode"; $sSQL .= "ST_SetSRID(ST_Point(x,y),4326) as geometry from us_postcode";