safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

aux properties, memory leaks, tweaks to search order

Browse Source
This commit is contained in:
Brian Quinion
2011-01-21 10:40:44 +00:00
parent 2be00f37c1
commit 31273a42bd
11 changed files with 401 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
nominatim/export.c
View File

@@ -161,7 +161,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_address",
"select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc",
"select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc,osm_type,osm_id",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
@@ -172,7 +172,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_names",
"select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x",
"select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x order by (each(name)).key",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
@@ -183,7 +183,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_extratags",
"select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x",
"select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x order by (each(extratags)).key",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
@@ -402,6 +402,7 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
PQclear(res);
PQclear(resNames);
PQclear(resAddress);
PQclear(resExtraTags);
}
const char * getRankLabel(int rank)

111
nominatim/import.c
View File

@@ -16,11 +16,11 @@
typedef enum { FILETYPE_NONE, FILETYPE_STRUCTUREDV0P1 } filetypes_t;
typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } filemodes_t;
#define MAX_FEATUREADDRESS 500
#define MAX_FEATURENAMES 1000
#define MAX_FEATUREEXTRATAGS 100
#define MAX_FEATURENAMESTRING 100000
#define MAX_FEATUREEXTRATAGSTRING 50000
#define MAX_FEATUREADDRESS 5000
#define MAX_FEATURENAMES 10000
#define MAX_FEATUREEXTRATAGS 10000
#define MAX_FEATURENAMESTRING 1000000
#define MAX_FEATUREEXTRATAGSTRING 500000
struct feature_address
{
@@ -42,7 +42,7 @@ struct feature_tag
struct feature
{
int placeID;
xmlChar * placeID;
xmlChar * type;
xmlChar * id;
xmlChar * key;
@@ -92,7 +92,7 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
}
else
{
fprintf( stderr, "Unknown osmStructured version %f\n", version );
fprintf( stderr, "Unknown osmStructured version %f (%s)\n", version, value );
exit_nicely();
}
}
@@ -127,6 +127,7 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
if (xmlStrEqual(name, BAD_CAST "feature"))
{
feature.placeID = xmlTextReaderGetAttribute(reader, BAD_CAST "place_id");
feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
@@ -140,32 +141,39 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
feature.geometry = NULL;
featureAddressLines = 0;
featureNameLines = 0;
featureExtraTagLines = 0;
return;
}
if (xmlStrEqual(name, BAD_CAST "names")) return;
if (xmlStrEqual(name, BAD_CAST "name"))
{
featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureName[featureNameLines].value = xmlTextReaderReadString(reader);
featureNameLines++;
if (featureNameLines >= MAX_FEATURENAMES)
if (featureNameLines < MAX_FEATURENAMES)
{
fprintf( stderr, "Too many name elements\n");
exit_nicely();
featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureName[featureNameLines].value = xmlTextReaderReadString(reader);
featureNameLines++;
}
else
{
fprintf( stderr, "Too many name elements (%s%s)\n", feature.type, feature.id);
// exit_nicely();
}
return;
}
if (xmlStrEqual(name, BAD_CAST "tags")) return;
if (xmlStrEqual(name, BAD_CAST "tag"))
{
featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
featureExtraTagLines++;
if (featureExtraTagLines >= MAX_FEATUREEXTRATAGS)
if (featureExtraTagLines < MAX_FEATUREEXTRATAGS)
{
featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
featureExtraTagLines++;
}
else
{
fprintf( stderr, "Too many extra tag elements\n");
exit_nicely();
fprintf( stderr, "Too many extra tag elements (%s%s)\n", feature.type, feature.id);
// exit_nicely();
}
return;
}
@@ -261,36 +269,39 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
}
if (isAddressLine)
{
value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
if (!value)
{
fprintf( stderr, "Address element missing rank\n");
exit_nicely();
}
featureAddress[featureAddressLines].rankAddress = atoi(value);
xmlFree(value);
if (featureAddressLines < MAX_FEATUREADDRESS)
{
value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
if (!value)
{
fprintf( stderr, "Address element missing rank\n");
exit_nicely();
}
featureAddress[featureAddressLines].rankAddress = atoi(value);
xmlFree(value);
value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
if (!value)
{
fprintf( stderr, "Address element missing rank\n");
exit_nicely();
}
if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
else strcpy(featureAddress[featureAddressLines].isAddress, "f");
xmlFree(value);
value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
if (!value)
{
fprintf( stderr, "Address element missing rank\n");
exit_nicely();
}
if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
else strcpy(featureAddress[featureAddressLines].isAddress, "f");
xmlFree(value);
featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
featureAddressLines++;
if (featureAddressLines >= MAX_FEATUREADDRESS)
featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
featureAddressLines++;
}
else
{
fprintf( stderr, "Too many address elements\n");
exit_nicely();
fprintf( stderr, "Too many address elements (%s%s)\n", feature.type, feature.id);
// exit_nicely();
}
return;
@@ -311,7 +322,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
{
featureCount++;
if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
/*
if (fileMode == FILEMODE_ADD)
{
resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
@@ -336,7 +347,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
exit(EXIT_FAILURE);
}
}
place_id = PQgetvalue(resPlaceID, 0, 0);
*/
place_id = feature.placeID;
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE)
{
@@ -517,6 +529,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
}
}
xmlFree(feature.placeID);
xmlFree(feature.type);
xmlFree(feature.id);
xmlFree(feature.key);
@@ -529,7 +542,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
if (feature.houseNumber) xmlFree(feature.houseNumber);
if (feature.geometry) xmlFree(feature.geometry);
PQclear(resPlaceID);
// PQclear(resPlaceID);
}
}

49
nominatim/index.c
View File

@@ -157,9 +157,9 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
paramValues[0] = (char *)&paramRank;
paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1;
if (rank < 16)
resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
else
// if (rank < 16)
// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
// else
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
{
@@ -226,10 +226,14 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
paramValues[1] = (char *)&paramSector;
paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1;
if (rank < 16)
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
if (rankTotalTuples-rankCountTuples < num_threads*20)
{
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
}
else
{
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
}
if (!iResult)
{
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
@@ -288,6 +292,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
PQclear(resPlaces);
}
if (rankTotalTuples-rankCountTuples < num_threads*20) iSector = PQntuples(resSectors);
}
// Finished rank
printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
@@ -331,16 +336,30 @@ void *nominatim_indexThread(void * thread_data_in)
if (verbose) printf(" Processing place_id %d\n", place_id);
updateStartTime = time(0);
paramPlaceID = PGint32(place_id);
paramValues[0] = (char *)&paramPlaceID;
paramLengths[0] = sizeof(paramPlaceID);
paramFormats[0] = 1;
res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
PQclear(res);
exit(EXIT_FAILURE);
int done = 0;
while(!done)
{
paramPlaceID = PGint32(place_id);
paramValues[0] = (char *)&paramPlaceID;
paramLengths[0] = sizeof(paramPlaceID);
paramFormats[0] = 1;
res = PQexecPrepared(thread_data->conn, "index_placex", 1, paramValues, paramLengths, paramFormats, 1);
if (PQresultStatus(res) == PGRES_COMMAND_OK)
done = 1;
else
{
if (strncmp(PQerrorMessage(thread_data->conn), "ERROR: deadlock detected", 25))
{
fprintf(stderr, "index_placex: UPDATE failed - deadlock, retrying\n");
}
else
{
fprintf(stderr, "index_placex: UPDATE failed: %s", PQerrorMessage(thread_data->conn));
PQclear(res);
sleep(5);
// exit(EXIT_FAILURE);
}
}
}
PQclear(res);
if (difftime(time(0), updateStartTime) > 1) printf(" Slow place_id %d\n", place_id);

165
nominatim/partitionedtags.def
View File

@@ -1,2 +1,165 @@
amenity pub
amenity airport
amenity arts_centre
amenity atm
amenity auditorium
amenity bank
amenity bar
amenity bench
amenity bicycle_parking
amenity bicycle_rental
amenity brothel
amenity bureau_de_change
amenity bus_station
amenity cafe
amenity car_rental
amenity car_wash
amenity casino
amenity cinema
amenity clinic
amenity club
amenity college
amenity community_centre
amenity courthouse
amenity crematorium
amenity dentist
amenity doctors
amenity dormitory
amenity drinking_water
amenity driving_school
amenity embassy
amenity emergency_phone
amenity fast_food
amenity ferry_terminal
amenity fire_hydrant
amenity fire_station
amenity fountain
amenity fuel
amenity grave_yard
amenity hall
amenity health_centre
amenity hospital
amenity hotel
amenity hunting_stand
amenity ice_cream
amenity kindergarten
amenity library
amenity market
amenity marketplace
amenity nightclub
amenity nursery
amenity nursing_home
amenity office
amenity park
amenity parking
amenity pharmacy
amenity place_of_worship
amenity police
amenity post_box
amenity post_office
amenity preschool
amenity prison
amenity pub
amenity public_building
amenity public_market
amenity reception_area
amenity restaurant
amenity retirement_home
amenity sauna
amenity school
amenity shelter
amenity shop
amenity shopping
amenity social_club
amenity studio
amenity supermarket
amenity taxi
amenity telephone
amenity theatre
amenity toilets
amenity townhall
amenity university
amenity veterinary
amenity waste_basket
amenity wifi
amenity youth_centre
boundary administrative
building apartments
building block
building bunker
building chapel
building church
building commercial
building dormitory
building entrance
building faculty
building farm
building flats
building garage
building hospital
building hotel
building house
building industrial
building office
building public
building residential
building retail
building school
building shop
building stadium
building store
building terrace
building tower
building train_station
building university
highway bridleway
highway bus_stop
highway construction
highway cycleway
highway distance_marker
highway emergency_access_point
highway footway
highway gate
highway motorway_junction
highway path
highway pedestrian
highway platform
highway primary
highway primary_link
highway raceway
highway road
highway secondary
highway secondary_link
highway services
highway steps
highway tertiary
highway track
highway trail
highway trunk
highway trunk_link
highway unsurfaced
historic archaeological_site
historic battlefield
historic building
historic castle
historic church
historic house
historic icon
historic manor
historic memorial
historic mine
historic monument
historic museum
historic ruins
historic tower
historic wayside_cross
historic wayside_shrine
historic wreck
landuse cemetery
landuse commercial
landuse construction
landuse farm
landuse farmland
landuse farmyard
landuse forest
landuse grass
landuse industrial