hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-26 11:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

clean up formatting

Browse Source
This commit is contained in:
Brian Quinion
2010-12-10 16:13:07 +00:00
parent 1427753846
commit 52cdaba0f1
9 changed files with 1144 additions and 1074 deletions
Download Patch File Download Diff File Expand all files Collapse all files

537
nominatim/export.c
View File

@@ -22,16 +22,16 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch
{ {
xmlTextWriterPtr writer; xmlTextWriterPtr writer;
int rankTotalDone; int rankTotalDone;
PGconn *conn; PGconn *conn;
PGresult * res; PGresult * res;
PGresult * resSectors; PGresult * resSectors;
PGresult * resPlaces; PGresult * resPlaces;
int rank; int rank;
int i; int i;
int iSector; int iSector;
int tuples; int tuples;
const char *paramValues[2]; const char *paramValues[2];
@@ -43,35 +43,36 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch
Oid pg_prepare_params[2]; Oid pg_prepare_params[2];
conn = PQconnectdb(conninfo); conn = PQconnectdb(conninfo);
if (PQstatus(conn) != CONNECTION_OK) { if (PQstatus(conn) != CONNECTION_OK)
{
fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
pg_prepare_params[0] = PG_OID_INT4; pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(conn, "index_sectors", res = PQprepare(conn, "index_sectors",
"select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status = 0 group by geometry_sector order by geometry_sector", "select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status = 0 group by geometry_sector order by geometry_sector",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) exit(EXIT_FAILURE); if (PQresultStatus(res) != PGRES_COMMAND_OK) exit(EXIT_FAILURE);
PQclear(res); PQclear(res);
pg_prepare_params[0] = PG_OID_INT4; pg_prepare_params[0] = PG_OID_INT4;
pg_prepare_params[1] = PG_OID_INT4; pg_prepare_params[1] = PG_OID_INT4;
res = PQprepare(conn, "index_sector_places", res = PQprepare(conn, "index_sector_places",
"select place_id from placex where rank_search = $1 and geometry_sector = $2", "select place_id from placex where rank_search = $1 and geometry_sector = $2",
2, pg_prepare_params); 2, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) exit(EXIT_FAILURE); if (PQresultStatus(res) != PGRES_COMMAND_OK) exit(EXIT_FAILURE);
PQclear(res); PQclear(res);
nominatim_exportCreatePreparedQueries(conn); nominatim_exportCreatePreparedQueries(conn);
// Create the output file // Create the output file
writer = nominatim_exportXMLStart(structuredoutputfile); writer = nominatim_exportXMLStart(structuredoutputfile);
for (rank = rank_min; rank <= rank_max; rank++) for (rank = rank_min; rank <= rank_max; rank++)
{ {
printf("Starting rank %d\n", rank); printf("Starting rank %d\n", rank);
paramRank = PGint32(rank); paramRank = PGint32(rank);
paramValues[0] = (char *)&paramRank; paramValues[0] = (char *)&paramRank;
@@ -84,56 +85,56 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch
PQclear(resSectors); PQclear(resSectors);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (PQftype(resSectors, 0) != PG_OID_INT4) if (PQftype(resSectors, 0) != PG_OID_INT4)
{ {
fprintf(stderr, "Sector value has unexpected type\n"); fprintf(stderr, "Sector value has unexpected type\n");
PQclear(resSectors); PQclear(resSectors);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (PQftype(resSectors, 1) != PG_OID_INT8) if (PQftype(resSectors, 1) != PG_OID_INT8)
{ {
fprintf(stderr, "Sector value has unexpected type\n"); fprintf(stderr, "Sector value has unexpected type\n");
PQclear(resSectors); PQclear(resSectors);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
rankTotalDone = 0; rankTotalDone = 0;
for (iSector = 0; iSector < PQntuples(resSectors); iSector++) for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
{ {
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
// Get all the place_id's for this sector // Get all the place_id's for this sector
paramRank = PGint32(rank); paramRank = PGint32(rank);
paramValues[0] = (char *)&paramRank; paramValues[0] = (char *)&paramRank;
paramLengths[0] = sizeof(paramRank); paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1; paramFormats[0] = 1;
paramSector = PGint32(sector); paramSector = PGint32(sector);
paramValues[1] = (char *)&paramSector; paramValues[1] = (char *)&paramSector;
paramLengths[1] = sizeof(paramSector); paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1; paramFormats[1] = 1;
resPlaces = PQexecPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1); resPlaces = PQexecPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK) if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces); PQclear(resPlaces);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (PQftype(resPlaces, 0) != PG_OID_INT4) if (PQftype(resPlaces, 0) != PG_OID_INT4)
{ {
fprintf(stderr, "Place_id value has unexpected type\n"); fprintf(stderr, "Place_id value has unexpected type\n");
PQclear(resPlaces); PQclear(resPlaces);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
tuples = PQntuples(resPlaces); tuples = PQntuples(resPlaces);
for(i = 0; i < tuples; i++) for (i = 0; i < tuples; i++)
{ {
nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL); nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL);
rankTotalDone++; rankTotalDone++;
if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000); if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000);
} }
PQclear(resPlaces); PQclear(resPlaces);
} }
PQclear(resSectors); PQclear(resSectors);
} }
@@ -145,50 +146,50 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch
void nominatim_exportCreatePreparedQueries(PGconn * conn) void nominatim_exportCreatePreparedQueries(PGconn * conn)
{ {
Oid pg_prepare_params[2]; Oid pg_prepare_params[2];
PGresult * res; PGresult * res;
pg_prepare_params[0] = PG_OID_INT8; pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_details", res = PQprepare(conn, "placex_details",
"select osm_type, osm_id, class, type, name, housenumber, country_code, ST_AsText(geometry), admin_level, rank_address, rank_search from placex where place_id = $1", "select osm_type, osm_id, class, type, name, housenumber, country_code, ST_AsText(geometry), admin_level, rank_address, rank_search from placex where place_id = $1",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Error preparing placex_details: %s", PQerrorMessage(conn)); fprintf(stderr, "Error preparing placex_details: %s", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
pg_prepare_params[0] = PG_OID_INT8; pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_address", res = PQprepare(conn, "placex_address",
"select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc", "select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Error preparing placex_address: %s", PQerrorMessage(conn)); fprintf(stderr, "Error preparing placex_address: %s", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
pg_prepare_params[0] = PG_OID_INT8; pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_names", res = PQprepare(conn, "placex_names",
"select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x", "select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Error preparing placex_names: %s", PQerrorMessage(conn)); fprintf(stderr, "Error preparing placex_names: %s", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
pg_prepare_params[0] = PG_OID_INT8; pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_extratags", res = PQprepare(conn, "placex_extratags",
"select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x", "select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Error preparing placex_extratags: %s", PQerrorMessage(conn)); fprintf(stderr, "Error preparing placex_extratags: %s", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
} }
@@ -197,36 +198,36 @@ xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile)
xmlTextWriterPtr writer; xmlTextWriterPtr writer;
writer = xmlNewTextWriterFilename(structuredoutputfile, 0); writer = xmlNewTextWriterFilename(structuredoutputfile, 0);
if (writer==NULL) if (writer==NULL)
{ {
fprintf(stderr, "Unable to open %s\n", structuredoutputfile); fprintf(stderr, "Unable to open %s\n", structuredoutputfile);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
xmlTextWriterSetIndent(writer, 1); xmlTextWriterSetIndent(writer, 1);
if (xmlTextWriterStartDocument(writer, NULL, "UTF8", NULL) < 0) if (xmlTextWriterStartDocument(writer, NULL, "UTF8", NULL) < 0)
{ {
fprintf(stderr, "xmlTextWriterStartDocument failed\n"); fprintf(stderr, "xmlTextWriterStartDocument failed\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (xmlTextWriterStartElement(writer, BAD_CAST "osmStructured") < 0) if (xmlTextWriterStartElement(writer, BAD_CAST "osmStructured") < 0)
{ {
fprintf(stderr, "xmlTextWriterStartElement failed\n"); fprintf(stderr, "xmlTextWriterStartElement failed\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (xmlTextWriterWriteAttribute(writer, BAD_CAST "version", BAD_CAST "0.1") < 0) if (xmlTextWriterWriteAttribute(writer, BAD_CAST "version", BAD_CAST "0.1") < 0)
{ {
fprintf(stderr, "xmlTextWriterWriteAttribute failed\n"); fprintf(stderr, "xmlTextWriterWriteAttribute failed\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (xmlTextWriterWriteAttribute(writer, BAD_CAST "generator", BAD_CAST "Nominatim") < 0) if (xmlTextWriterWriteAttribute(writer, BAD_CAST "generator", BAD_CAST "Nominatim") < 0)
{ {
fprintf(stderr, "xmlTextWriterWriteAttribute failed\n"); fprintf(stderr, "xmlTextWriterWriteAttribute failed\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0) if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
{ {
fprintf(stderr, "xmlTextWriterStartElement failed\n"); fprintf(stderr, "xmlTextWriterStartElement failed\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
return writer; return writer;
@@ -234,22 +235,22 @@ xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile)
void nominatim_exportXMLEnd(xmlTextWriterPtr writer) void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
{ {
// End <add> // End <add>
if (xmlTextWriterEndElement(writer) < 0) if (xmlTextWriterEndElement(writer) < 0)
{ {
fprintf(stderr, "xmlTextWriterEndElement failed\n"); fprintf(stderr, "xmlTextWriterEndElement failed\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
// End <osmStructured> // End <osmStructured>
if (xmlTextWriterEndElement(writer) < 0) if (xmlTextWriterEndElement(writer) < 0)
{ {
fprintf(stderr, "xmlTextWriterEndElement failed\n"); fprintf(stderr, "xmlTextWriterEndElement failed\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (xmlTextWriterEndDocument(writer) < 0) if (xmlTextWriterEndDocument(writer) < 0)
{ {
fprintf(stderr, "xmlTextWriterEndDocument failed\n"); fprintf(stderr, "xmlTextWriterEndDocument failed\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
xmlFreeTextWriter(writer); xmlFreeTextWriter(writer);
} }
@@ -259,12 +260,12 @@ void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
*/ */
void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex) void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex)
{ {
PGresult * res; PGresult * res;
PGresult * resNames; PGresult * resNames;
PGresult * resAddress; PGresult * resAddress;
PGresult * resExtraTags; PGresult * resExtraTags;
int i; int i;
const char * paramValues[1]; const char * paramValues[1];
int paramLengths[1]; int paramLengths[1];
@@ -272,190 +273,190 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
uint64_t paramPlaceID; uint64_t paramPlaceID;
paramPlaceID = PGint64(place_id); paramPlaceID = PGint64(place_id);
paramValues[0] = (char *)&paramPlaceID; paramValues[0] = (char *)&paramPlaceID;
paramLengths[0] = sizeof(paramPlaceID); paramLengths[0] = sizeof(paramPlaceID);
paramFormats[0] = 1; paramFormats[0] = 1;
res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0); res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(res) != PGRES_TUPLES_OK) if (PQresultStatus(res) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "placex_details: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "placex_details: SELECT failed: %s", PQerrorMessage(conn));
PQclear(res); PQclear(res);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0); resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(resNames) != PGRES_TUPLES_OK) if (PQresultStatus(resNames) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "placex_names: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "placex_names: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resNames); PQclear(resNames);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0); resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(resAddress) != PGRES_TUPLES_OK) if (PQresultStatus(resAddress) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "placex_address: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "placex_address: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resAddress); PQclear(resAddress);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0); resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(resExtraTags) != PGRES_TUPLES_OK) if (PQresultStatus(resExtraTags) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resExtraTags); PQclear(resExtraTags);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (writer_mutex) pthread_mutex_lock( writer_mutex ); if (writer_mutex) pthread_mutex_lock( writer_mutex );
xmlTextWriterStartElement(writer, BAD_CAST "feature"); xmlTextWriterStartElement(writer, BAD_CAST "feature");
xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id); xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(res, 0, 0)); xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(res, 0, 0));
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(res, 0, 1)); xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(res, 0, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(res, 0, 2)); xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(res, 0, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(res, 0, 3)); xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(res, 0, 3));
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(res, 0, 9)); xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(res, 0, 9));
xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(res, 0, 10)); xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(res, 0, 10));
if (PQntuples(resNames)) if (PQntuples(resNames))
{ {
xmlTextWriterStartElement(writer, BAD_CAST "names"); xmlTextWriterStartElement(writer, BAD_CAST "names");
for(i = 0; i < PQntuples(resNames); i++) for (i = 0; i < PQntuples(resNames); i++)
{ {
xmlTextWriterStartElement(writer, BAD_CAST "name"); xmlTextWriterStartElement(writer, BAD_CAST "name");
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resNames, i, 0)); xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resNames, i, 0));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resNames, i, 1)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resNames, i, 1));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
if (PQgetvalue(res, 0, 5) && strlen(PQgetvalue(res, 0, 5))) if (PQgetvalue(res, 0, 5) && strlen(PQgetvalue(res, 0, 5)))
{ {
xmlTextWriterStartElement(writer, BAD_CAST "houseNumber"); xmlTextWriterStartElement(writer, BAD_CAST "houseNumber");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 5)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 5));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
if (PQgetvalue(res, 0, 8) && strlen(PQgetvalue(res, 0, 8))) if (PQgetvalue(res, 0, 8) && strlen(PQgetvalue(res, 0, 8)))
{ {
xmlTextWriterStartElement(writer, BAD_CAST "adminLevel"); xmlTextWriterStartElement(writer, BAD_CAST "adminLevel");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 8)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 8));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
if (PQgetvalue(res, 0, 6) && strlen(PQgetvalue(res, 0, 6))) if (PQgetvalue(res, 0, 6) && strlen(PQgetvalue(res, 0, 6)))
{ {
xmlTextWriterStartElement(writer, BAD_CAST "countryCode"); xmlTextWriterStartElement(writer, BAD_CAST "countryCode");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 6)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 6));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
if (PQntuples(resAddress) > 0) if (PQntuples(resAddress) > 0)
{ {
xmlTextWriterStartElement(writer, BAD_CAST "address"); xmlTextWriterStartElement(writer, BAD_CAST "address");
for(i = 0; i < PQntuples(resAddress); i++) for (i = 0; i < PQntuples(resAddress); i++)
{ {
xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(resAddress, i, 5)))); xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(resAddress, i, 5))));
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(resAddress, i, 5)); xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(resAddress, i, 5));
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resAddress, i, 0)); xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resAddress, i, 0));
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(resAddress, i, 1)); xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(resAddress, i, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(resAddress, i, 2)); xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(resAddress, i, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(resAddress, i, 3)); xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(resAddress, i, 3));
xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(resAddress, i, 4)); xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(resAddress, i, 4));
xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(resAddress, i, 6)); xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(resAddress, i, 6));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
if (PQntuples(resExtraTags)) if (PQntuples(resExtraTags))
{ {
xmlTextWriterStartElement(writer, BAD_CAST "tags"); xmlTextWriterStartElement(writer, BAD_CAST "tags");
for(i = 0; i < PQntuples(resExtraTags); i++) for (i = 0; i < PQntuples(resExtraTags); i++)
{ {
xmlTextWriterStartElement(writer, BAD_CAST "tag"); xmlTextWriterStartElement(writer, BAD_CAST "tag");
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resExtraTags, i, 0)); xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resExtraTags, i, 0));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resExtraTags, i, 1)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resExtraTags, i, 1));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
} }
xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry"); xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 7)); xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 7));
xmlTextWriterEndElement(writer); xmlTextWriterEndElement(writer);
xmlTextWriterEndElement(writer); // </feature> xmlTextWriterEndElement(writer); // </feature>
if (writer_mutex) pthread_mutex_unlock( writer_mutex ); if (writer_mutex) pthread_mutex_unlock( writer_mutex );
PQclear(res); PQclear(res);
PQclear(resNames); PQclear(resNames);
PQclear(resAddress); PQclear(resAddress);
} }
const char * getRankLabel(int rank) const char * getRankLabel(int rank)
{ {
switch(rank) switch (rank)
{ {
case 0: case 0:
case 1: case 1:
return "continent"; return "continent";
case 2: case 2:
case 3: case 3:
return "sea"; return "sea";
case 4: case 4:
case 5: case 5:
case 6: case 6:
case 7: case 7:
return "country"; return "country";
case 8: case 8:
case 9: case 9:
case 10: case 10:
case 11: case 11:
return "state"; return "state";
case 12: case 12:
case 13: case 13:
case 14: case 14:
case 15: case 15:
return "county"; return "county";
case 16: case 16:
return "city"; return "city";
case 17: case 17:
return "town"; return "town";
case 18: case 18:
return "village"; return "village";
case 19: case 19:
return "unknown"; return "unknown";
case 20: case 20:
return "suburb"; return "suburb";
case 21: case 21:
return "postcode"; return "postcode";
case 22: case 22:
return "neighborhood"; return "neighborhood";
case 23: case 23:
return "postcode"; return "postcode";
case 24: case 24:
return "unknown"; return "unknown";
case 25: case 25:
return "postcode"; return "postcode";
case 26: case 26:
return "street"; return "street";
case 27: case 27:
return "access"; return "access";
case 28: case 28:
return "building"; return "building";
case 29: case 29:
default: default:
return "other"; return "other";
} }
} }

740
nominatim/import.c
View File

@@ -22,34 +22,37 @@ typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } f
#define MAX_FEATURENAMESTRING 100000 #define MAX_FEATURENAMESTRING 100000
#define MAX_FEATUREEXTRATAGSTRING 50000 #define MAX_FEATUREEXTRATAGSTRING 50000
struct feature_address { struct feature_address
int place_id; {
int rankAddress; int place_id;
char isAddress[2]; int rankAddress;
xmlChar * type; char isAddress[2];
xmlChar * id; xmlChar * type;
xmlChar * key; xmlChar * id;
xmlChar * value; xmlChar * key;
xmlChar * distance; xmlChar * value;
xmlChar * distance;
}; };
struct feature_tag { struct feature_tag
xmlChar * type; {
xmlChar * value; xmlChar * type;
xmlChar * value;
}; };
struct feature { struct feature
int placeID; {
xmlChar * type; int placeID;
xmlChar * id; xmlChar * type;
xmlChar * key; xmlChar * id;
xmlChar * value; xmlChar * key;
xmlChar * rankAddress; xmlChar * value;
xmlChar * rankSearch; xmlChar * rankAddress;
xmlChar * countryCode; xmlChar * rankSearch;
xmlChar * adminLevel; xmlChar * countryCode;
xmlChar * houseNumber; xmlChar * adminLevel;
xmlChar * geometry; xmlChar * houseNumber;
xmlChar * geometry;
} feature; } feature;
int fileType = FILETYPE_NONE; int fileType = FILETYPE_NONE;
@@ -75,23 +78,23 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
if (fileType == FILETYPE_NONE) if (fileType == FILETYPE_NONE)
{ {
// Potential to handle other file types in the future / versions // Potential to handle other file types in the future / versions
if (xmlStrEqual(name, BAD_CAST "osmStructured")) if (xmlStrEqual(name, BAD_CAST "osmStructured"))
{ {
value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "version"); value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "version");
version = strtof(value, NULL); version = strtof(value, NULL);
xmlFree(value); xmlFree(value);
if (version == (float)0.1) if (version == (float)0.1)
{ {
fileType = FILETYPE_STRUCTUREDV0P1; fileType = FILETYPE_STRUCTUREDV0P1;
fileMode = FILEMODE_ADD; fileMode = FILEMODE_ADD;
} }
else else
{ {
fprintf( stderr, "Unknown osmStructured version %f\n", version ); fprintf( stderr, "Unknown osmStructured version %f\n", version );
exit_nicely(); exit_nicely();
} }
} }
else else
{ {
@@ -103,18 +106,18 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
if (xmlStrEqual(name, BAD_CAST "add")) if (xmlStrEqual(name, BAD_CAST "add"))
{ {
fileMode = FILEMODE_ADD; fileMode = FILEMODE_ADD;
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "update")) if (xmlStrEqual(name, BAD_CAST "update"))
{ {
fileMode = FILEMODE_UPDATE; fileMode = FILEMODE_UPDATE;
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "delete")) if (xmlStrEqual(name, BAD_CAST "delete"))
{ {
fileMode = FILEMODE_DELETE; fileMode = FILEMODE_DELETE;
return; return;
} }
if (fileMode == FILEMODE_NONE) if (fileMode == FILEMODE_NONE)
{ {
@@ -124,72 +127,72 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
if (xmlStrEqual(name, BAD_CAST "feature")) if (xmlStrEqual(name, BAD_CAST "feature"))
{ {
feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type"); feature.type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); feature.id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key"); feature.key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
feature.value = xmlTextReaderGetAttribute(reader, BAD_CAST "value"); feature.value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank"); feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance"); feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
feature.countryCode = NULL; feature.countryCode = NULL;
feature.adminLevel = NULL; feature.adminLevel = NULL;
feature.houseNumber = NULL; feature.houseNumber = NULL;
feature.geometry = NULL; feature.geometry = NULL;
featureAddressLines = 0; featureAddressLines = 0;
featureNameLines = 0; featureNameLines = 0;
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "names")) return; if (xmlStrEqual(name, BAD_CAST "names")) return;
if (xmlStrEqual(name, BAD_CAST "name")) if (xmlStrEqual(name, BAD_CAST "name"))
{ {
featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type"); featureName[featureNameLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureName[featureNameLines].value = xmlTextReaderReadString(reader); featureName[featureNameLines].value = xmlTextReaderReadString(reader);
featureNameLines++; featureNameLines++;
if (featureNameLines >= MAX_FEATURENAMES) if (featureNameLines >= MAX_FEATURENAMES)
{ {
fprintf( stderr, "Too many name elements\n"); fprintf( stderr, "Too many name elements\n");
exit_nicely(); exit_nicely();
} }
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "tags")) return; if (xmlStrEqual(name, BAD_CAST "tags")) return;
if (xmlStrEqual(name, BAD_CAST "tag")) if (xmlStrEqual(name, BAD_CAST "tag"))
{ {
featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type"); featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader); featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
featureExtraTagLines++; featureExtraTagLines++;
if (featureExtraTagLines >= MAX_FEATUREEXTRATAGS) if (featureExtraTagLines >= MAX_FEATUREEXTRATAGS)
{ {
fprintf( stderr, "Too many extra tag elements\n"); fprintf( stderr, "Too many extra tag elements\n");
exit_nicely(); exit_nicely();
} }
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "osmGeometry")) if (xmlStrEqual(name, BAD_CAST "osmGeometry"))
{ {
feature.geometry = xmlTextReaderReadString(reader); feature.geometry = xmlTextReaderReadString(reader);
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "adminLevel")) if (xmlStrEqual(name, BAD_CAST "adminLevel"))
{ {
feature.adminLevel = xmlTextReaderReadString(reader); feature.adminLevel = xmlTextReaderReadString(reader);
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "countryCode")) if (xmlStrEqual(name, BAD_CAST "countryCode"))
{ {
feature.countryCode = xmlTextReaderReadString(reader); feature.countryCode = xmlTextReaderReadString(reader);
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "houseNumber")) if (xmlStrEqual(name, BAD_CAST "houseNumber"))
{ {
feature.houseNumber = xmlTextReaderReadString(reader); feature.houseNumber = xmlTextReaderReadString(reader);
return; return;
} }
if (xmlStrEqual(name, BAD_CAST "address")) if (xmlStrEqual(name, BAD_CAST "address"))
{ {
featureAddressLines = 0; featureAddressLines = 0;
return; return;
} }
isAddressLine = 0; isAddressLine = 0;
if (xmlStrEqual(name, BAD_CAST "continent")) if (xmlStrEqual(name, BAD_CAST "continent"))
@@ -258,66 +261,66 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
} }
if (isAddressLine) if (isAddressLine)
{ {
value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank"); value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
if (!value) if (!value)
{ {
fprintf( stderr, "Address element missing rank\n"); fprintf( stderr, "Address element missing rank\n");
exit_nicely(); exit_nicely();
} }
featureAddress[featureAddressLines].rankAddress = atoi(value); featureAddress[featureAddressLines].rankAddress = atoi(value);
xmlFree(value); xmlFree(value);
value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress"); value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
if (!value) if (!value)
{ {
fprintf( stderr, "Address element missing rank\n"); fprintf( stderr, "Address element missing rank\n");
exit_nicely(); exit_nicely();
} }
if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t"); if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
else strcpy(featureAddress[featureAddressLines].isAddress, "f"); else strcpy(featureAddress[featureAddressLines].isAddress, "f");
xmlFree(value); xmlFree(value);
featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type"); featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id"); featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key"); featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value"); featureAddress[featureAddressLines].value = xmlTextReaderGetAttribute(reader, BAD_CAST "value");
featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance"); featureAddress[featureAddressLines].distance = xmlTextReaderGetAttribute(reader, BAD_CAST "distance");
featureAddressLines++; featureAddressLines++;
if (featureAddressLines >= MAX_FEATUREADDRESS) if (featureAddressLines >= MAX_FEATUREADDRESS)
{ {
fprintf( stderr, "Too many address elements\n"); fprintf( stderr, "Too many address elements\n");
exit_nicely(); exit_nicely();
} }
return; return;
} }
fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name); fprintf(stderr, "%s: Unknown element name: %s\n", __FUNCTION__, name);
} }
void EndElement(xmlTextReaderPtr reader, const xmlChar *name) void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
{ {
PGresult * res; PGresult * res;
PGresult * resPlaceID; PGresult * resPlaceID;
const char * paramValues[11]; const char * paramValues[11];
char * place_id; char * place_id;
char * partionQueryName; char * partionQueryName;
int i, namePos, lineTypeLen, lineValueLen; int i, namePos, lineTypeLen, lineValueLen;
if (xmlStrEqual(name, BAD_CAST "feature")) if (xmlStrEqual(name, BAD_CAST "feature"))
{ {
featureCount++; featureCount++;
if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000); if (featureCount % 1000 == 0) printf("feature %i(k)\n", featureCount/1000);
if (fileMode == FILEMODE_ADD) if (fileMode == FILEMODE_ADD)
{ {
resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0); resPlaceID = PQexecPrepared(conn, "get_new_place_id", 0, NULL, NULL, NULL, 0);
if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK) if (PQresultStatus(resPlaceID) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "get_place_id: INSERT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "get_place_id: INSERT failed: %s", PQerrorMessage(conn));
PQclear(resPlaceID); PQclear(resPlaceID);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
else else
{ {
@@ -333,7 +336,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
place_id = PQgetvalue(resPlaceID, 0, 0); place_id = PQgetvalue(resPlaceID, 0, 0);
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE) if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE)
{ {
@@ -368,195 +371,195 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD) if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
{ {
// Insert into placex // Insert into placex
paramValues[0] = (const char *)place_id; paramValues[0] = (const char *)place_id;
paramValues[1] = (const char *)feature.type; paramValues[1] = (const char *)feature.type;
paramValues[2] = (const char *)feature.id; paramValues[2] = (const char *)feature.id;
paramValues[3] = (const char *)feature.key; paramValues[3] = (const char *)feature.key;
paramValues[4] = (const char *)feature.value; paramValues[4] = (const char *)feature.value;
featureNameString[0] = 0; featureNameString[0] = 0;
if (featureNameLines) if (featureNameLines)
{ {
namePos = 0; namePos = 0;
lineTypeLen = 0; lineTypeLen = 0;
lineValueLen = 0; lineValueLen = 0;
for(i = 0; i < featureNameLines; i++) for (i = 0; i < featureNameLines; i++)
{ {
lineTypeLen = strlen(BAD_CAST featureName[i].type); lineTypeLen = strlen(BAD_CAST featureName[i].type);
lineValueLen = strlen(BAD_CAST featureName[i].value); lineValueLen = strlen(BAD_CAST featureName[i].value);
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING) if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
{ {
fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value); fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
break; break;
} }
if (namePos) strcpy(featureNameString+(namePos++), ","); if (namePos) strcpy(featureNameString+(namePos++), ",");
strcpy(featureNameString+(namePos++), "\""); strcpy(featureNameString+(namePos++), "\"");
strcpy(featureNameString+namePos, BAD_CAST featureName[i].type); strcpy(featureNameString+namePos, BAD_CAST featureName[i].type);
namePos += lineTypeLen; namePos += lineTypeLen;
strcpy(featureNameString+namePos, "\"=>\""); strcpy(featureNameString+namePos, "\"=>\"");
namePos += 4; namePos += 4;
strcpy(featureNameString+namePos, BAD_CAST featureName[i].value); strcpy(featureNameString+namePos, BAD_CAST featureName[i].value);
namePos += lineValueLen; namePos += lineValueLen;
strcpy(featureNameString+(namePos++), "\""); strcpy(featureNameString+(namePos++), "\"");
} }
} }
paramValues[5] = (const char *)featureNameString; paramValues[5] = (const char *)featureNameString;
featureExtraTagString[0] = 0; featureExtraTagString[0] = 0;
if (featureExtraTagLines) if (featureExtraTagLines)
{ {
namePos = 0; namePos = 0;
lineTypeLen = 0; lineTypeLen = 0;
lineValueLen = 0; lineValueLen = 0;
for(i = 0; i < featureExtraTagLines; i++) for (i = 0; i < featureExtraTagLines; i++)
{ {
lineTypeLen = strlen(BAD_CAST featureExtraTag[i].type); lineTypeLen = strlen(BAD_CAST featureExtraTag[i].type);
lineValueLen = strlen(BAD_CAST featureExtraTag[i].value); lineValueLen = strlen(BAD_CAST featureExtraTag[i].value);
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING) if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
{ {
fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value); fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
break; break;
} }
if (namePos) strcpy(featureExtraTagString+(namePos++),","); if (namePos) strcpy(featureExtraTagString+(namePos++),",");
strcpy(featureExtraTagString+(namePos++), "\""); strcpy(featureExtraTagString+(namePos++), "\"");
strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].type); strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].type);
namePos += lineTypeLen; namePos += lineTypeLen;
strcpy(featureExtraTagString+namePos, "\"=>\""); strcpy(featureExtraTagString+namePos, "\"=>\"");
namePos += 4; namePos += 4;
strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].value); strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].value);
namePos += lineValueLen; namePos += lineValueLen;
strcpy(featureExtraTagString+(namePos++), "\""); strcpy(featureExtraTagString+(namePos++), "\"");
} }
} }
paramValues[6] = (const char *)featureExtraTagString; paramValues[6] = (const char *)featureExtraTagString;
paramValues[7] = (const char *)feature.adminLevel; paramValues[7] = (const char *)feature.adminLevel;
paramValues[8] = (const char *)feature.houseNumber; paramValues[8] = (const char *)feature.houseNumber;
paramValues[9] = (const char *)feature.rankAddress; paramValues[9] = (const char *)feature.rankAddress;
paramValues[10] = (const char *)feature.rankSearch; paramValues[10] = (const char *)feature.rankSearch;
paramValues[11] = (const char *)feature.geometry; paramValues[11] = (const char *)feature.geometry;
res = PQexecPrepared(conn, "placex_insert", 12, paramValues, NULL, NULL, 0); res = PQexecPrepared(conn, "placex_insert", 12, paramValues, NULL, NULL, 0);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
PQclear(res); PQclear(res);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
for(i = 0; i < featureAddressLines; i++) for (i = 0; i < featureAddressLines; i++)
{ {
// insert into place_address // insert into place_address
paramValues[0] = (const char *)place_id; paramValues[0] = (const char *)place_id;
paramValues[1] = (const char *)featureAddress[i].distance; paramValues[1] = (const char *)featureAddress[i].distance;
paramValues[2] = (const char *)featureAddress[i].type; paramValues[2] = (const char *)featureAddress[i].type;
paramValues[3] = (const char *)featureAddress[i].id; paramValues[3] = (const char *)featureAddress[i].id;
paramValues[4] = (const char *)featureAddress[i].key; paramValues[4] = (const char *)featureAddress[i].key;
paramValues[5] = (const char *)featureAddress[i].value; paramValues[5] = (const char *)featureAddress[i].value;
paramValues[6] = (const char *)featureAddress[i].isAddress; paramValues[6] = (const char *)featureAddress[i].isAddress;
res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0); res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
PQclear(res); PQclear(res);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
xmlFree(featureAddress[i].type); xmlFree(featureAddress[i].type);
xmlFree(featureAddress[i].id); xmlFree(featureAddress[i].id);
xmlFree(featureAddress[i].key); xmlFree(featureAddress[i].key);
xmlFree(featureAddress[i].value); xmlFree(featureAddress[i].value);
xmlFree(featureAddress[i].distance); xmlFree(featureAddress[i].distance);
} }
if (featureNameLines) if (featureNameLines)
{ {
paramValues[0] = (const char *)place_id; paramValues[0] = (const char *)place_id;
res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0); res = PQexecPrepared(conn, "search_name_insert", 1, paramValues, NULL, NULL, 0);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "search_name_insert: INSERT failed: %s", PQerrorMessage(conn));
PQclear(res); PQclear(res);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
} }
partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value); partionQueryName = xmlHashLookup2(partionTableTagsHash, feature.key, feature.value);
if (partionQueryName) if (partionQueryName)
{ {
// insert into partition table // insert into partition table
paramValues[0] = (const char *)place_id; paramValues[0] = (const char *)place_id;
paramValues[1] = (const char *)feature.geometry; paramValues[1] = (const char *)feature.geometry;
res = PQexecPrepared(conn, partionQueryName, 2, paramValues, NULL, NULL, 0); res = PQexecPrepared(conn, partionQueryName, 2, paramValues, NULL, NULL, 0);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "%s: INSERT failed: %s", partionQueryName, PQerrorMessage(conn)); fprintf(stderr, "%s: INSERT failed: %s", partionQueryName, PQerrorMessage(conn));
PQclear(res); PQclear(res);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
} }
} }
else else
{ {
for(i = 0; i < featureAddressLines; i++) for (i = 0; i < featureAddressLines; i++)
{ {
xmlFree(featureAddress[i].type); xmlFree(featureAddress[i].type);
xmlFree(featureAddress[i].id); xmlFree(featureAddress[i].id);
xmlFree(featureAddress[i].key); xmlFree(featureAddress[i].key);
xmlFree(featureAddress[i].value); xmlFree(featureAddress[i].value);
xmlFree(featureAddress[i].distance); xmlFree(featureAddress[i].distance);
} }
} }
xmlFree(feature.type); xmlFree(feature.type);
xmlFree(feature.id); xmlFree(feature.id);
xmlFree(feature.key); xmlFree(feature.key);
xmlFree(feature.value); xmlFree(feature.value);
xmlFree(feature.rankAddress); xmlFree(feature.rankAddress);
xmlFree(feature.rankSearch); xmlFree(feature.rankSearch);
// if (feature.name) xmlFree(feature.name); // if (feature.name) xmlFree(feature.name);
if (feature.countryCode) xmlFree(feature.countryCode); if (feature.countryCode) xmlFree(feature.countryCode);
if (feature.adminLevel) xmlFree(feature.adminLevel); if (feature.adminLevel) xmlFree(feature.adminLevel);
if (feature.houseNumber) xmlFree(feature.houseNumber); if (feature.houseNumber) xmlFree(feature.houseNumber);
if (feature.geometry) xmlFree(feature.geometry); if (feature.geometry) xmlFree(feature.geometry);
PQclear(resPlaceID); PQclear(resPlaceID);
} }
} }
static void processNode(xmlTextReaderPtr reader) static void processNode(xmlTextReaderPtr reader)
{ {
xmlChar *name; xmlChar *name;
name = xmlTextReaderName(reader); name = xmlTextReaderName(reader);
if (name == NULL) if (name == NULL)
{ {
name = xmlStrdup(BAD_CAST "--"); name = xmlStrdup(BAD_CAST "--");
} }
switch(xmlTextReaderNodeType(reader)) switch (xmlTextReaderNodeType(reader))
{ {
case XML_READER_TYPE_ELEMENT: case XML_READER_TYPE_ELEMENT:
StartElement(reader, name); StartElement(reader, name);
if (xmlTextReaderIsEmptyElement(reader)) if (xmlTextReaderIsEmptyElement(reader))
EndElement(reader, name); /* No end_element for self closing tags! */ EndElement(reader, name); /* No end_element for self closing tags! */
break; break;
case XML_READER_TYPE_END_ELEMENT: case XML_READER_TYPE_END_ELEMENT:
EndElement(reader, name); EndElement(reader, name);
break; break;
case XML_READER_TYPE_TEXT: case XML_READER_TYPE_TEXT:
case XML_READER_TYPE_CDATA: case XML_READER_TYPE_CDATA:
case XML_READER_TYPE_SIGNIFICANT_WHITESPACE: case XML_READER_TYPE_SIGNIFICANT_WHITESPACE:
/* Ignore */ /* Ignore */
break; break;
default: default:
fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader)); fprintf(stderr, "Unknown node type %d\n", xmlTextReaderNodeType(reader));
break; break;
} }
xmlFree(name); xmlFree(name);
@@ -566,139 +569,139 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
{ {
xmlTextReaderPtr reader; xmlTextReaderPtr reader;
int ret = 0; int ret = 0;
PGresult * res; PGresult * res;
FILE * partionTagsFile; FILE * partionTagsFile;
char * partionQueryName; char * partionQueryName;
char partionQuerySQL[1024]; char partionQuerySQL[1024];
conn = PQconnectdb(conninfo); conn = PQconnectdb(conninfo);
if (PQstatus(conn) != CONNECTION_OK) if (PQstatus(conn) != CONNECTION_OK)
{ {
fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
partionTableTagsHash = xmlHashCreate(200); partionTableTagsHash = xmlHashCreate(200);
partionTagsFile = fopen(partionTagsFilename, "rt"); partionTagsFile = fopen(partionTagsFilename, "rt");
if (!partionTagsFile) if (!partionTagsFile)
{ {
fprintf(stderr, "Unable to read partition tags file: %s\n", partionTagsFilename); fprintf(stderr, "Unable to read partition tags file: %s\n", partionTagsFilename);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
char buffer[1024], osmkey[256], osmvalue[256]; char buffer[1024], osmkey[256], osmvalue[256];
int fields; int fields;
while(fgets(buffer, sizeof(buffer), partionTagsFile) != NULL) while (fgets(buffer, sizeof(buffer), partionTagsFile) != NULL)
{ {
fields = sscanf( buffer, "%23s %63s", osmkey, osmvalue ); fields = sscanf( buffer, "%23s %63s", osmkey, osmvalue );
if( fields <= 0 ) continue; if ( fields <= 0 ) continue;
if( fields != 2 ) if ( fields != 2 )
{ {
fprintf( stderr, "Error partition file\n"); fprintf( stderr, "Error partition file\n");
exit_nicely(); exit_nicely();
} }
partionQueryName = malloc(strlen("partition_insert_")+strlen(osmkey)+strlen(osmvalue)+2); partionQueryName = malloc(strlen("partition_insert_")+strlen(osmkey)+strlen(osmvalue)+2);
strcpy(partionQueryName, "partition_insert_"); strcpy(partionQueryName, "partition_insert_");
strcat(partionQueryName, osmkey); strcat(partionQueryName, osmkey);
strcat(partionQueryName, "_"); strcat(partionQueryName, "_");
strcat(partionQueryName, osmvalue); strcat(partionQueryName, osmvalue);
strcpy(partionQuerySQL, "insert into place_classtype_"); strcpy(partionQuerySQL, "insert into place_classtype_");
strcat(partionQuerySQL, osmkey); strcat(partionQuerySQL, osmkey);
strcat(partionQuerySQL, "_"); strcat(partionQuerySQL, "_");
strcat(partionQuerySQL, osmvalue); strcat(partionQuerySQL, osmvalue);
strcat(partionQuerySQL, " (place_id, centroid) values ($1, ST_Centroid(st_setsrid($2, 4326)))"); strcat(partionQuerySQL, " (place_id, centroid) values ($1, ST_Centroid(st_setsrid($2, 4326)))");
res = PQprepare(conn, partionQueryName, partionQuerySQL, 2, NULL); res = PQprepare(conn, partionQueryName, partionQuerySQL, 2, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn)); fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName); xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
} }
res = PQprepare(conn, "get_new_place_id", res = PQprepare(conn, "get_new_place_id",
"select nextval('seq_place')", "select nextval('seq_place')",
0, NULL); 0, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed to prepare get_new_place_id: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed to prepare get_new_place_id: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
res = PQprepare(conn, "get_place_id", res = PQprepare(conn, "get_place_id",
"select place_id from placex where osm_type = $1 and osm_id = $2 and class = $3 and type = $4", "select place_id from placex where osm_type = $1 and osm_id = $2 and class = $3 and type = $4",
4, NULL); 4, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed to prepare get_place_id: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed to prepare get_place_id: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
res = PQprepare(conn, "placex_insert", res = PQprepare(conn, "placex_insert",
"insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,admin_level,housenumber,rank_address,rank_search,geometry) " "insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,admin_level,housenumber,rank_address,rank_search,geometry) "
"values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, st_setsrid($12, 4326))", "values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, st_setsrid($12, 4326))",
12, NULL); 12, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
res = PQprepare(conn, "search_name_insert", res = PQprepare(conn, "search_name_insert",
"insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) " "insert into search_name (place_id, search_rank, address_rank, country_code, name_vector, nameaddress_vector, centroid) "
"select place_id, rank_address, rank_search, country_code, make_keywords(name), " "select place_id, rank_address, rank_search, country_code, make_keywords(name), "
"(select uniq(sort(array_agg(name_vector))) from place_addressline join search_name on " "(select uniq(sort(array_agg(name_vector))) from place_addressline join search_name on "
"(address_place_id = search_name.place_id) where place_addressline.place_id = $1 ), st_centroid(geometry) from placex " "(address_place_id = search_name.place_id) where place_addressline.place_id = $1 ), st_centroid(geometry) from placex "
"where place_id = $1", "where place_id = $1",
1, NULL); 1, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
res = PQprepare(conn, "place_addressline_insert", res = PQprepare(conn, "place_addressline_insert",
"insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) " "insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) "
"select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6", "select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
7, NULL); 7, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
res = PQprepare(conn, "placex_delete", res = PQprepare(conn, "placex_delete",
"delete from placex where place_id = $1", "delete from placex where place_id = $1",
1, NULL); 1, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed to prepare placex_delete: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed to prepare placex_delete: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
res = PQprepare(conn, "search_name_delete", res = PQprepare(conn, "search_name_delete",
"delete from search_name where place_id = $1", "delete from search_name where place_id = $1",
1, NULL); 1, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed to prepare search_name_delete: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed to prepare search_name_delete: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
res = PQprepare(conn, "place_addressline_delete", res = PQprepare(conn, "place_addressline_delete",
"delete from place_addressline where place_id = $1", "delete from place_addressline where place_id = $1",
1, NULL); 1, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed to prepare place_addressline_delete: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed to prepare place_addressline_delete: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
featureCount = 0; featureCount = 0;
@@ -713,16 +716,17 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
ret = xmlTextReaderRead(reader); ret = xmlTextReaderRead(reader);
while (ret == 1) while (ret == 1)
{ {
processNode(reader); processNode(reader);
ret = xmlTextReaderRead(reader); ret = xmlTextReaderRead(reader);
}
if (ret != 0)
{
fprintf(stderr, "%s : failed to parse\n", filename);
return ret;
} }
if (ret != 0) {
fprintf(stderr, "%s : failed to parse\n", filename);
return ret;
}
xmlFreeTextReader(reader); xmlFreeTextReader(reader);
xmlHashFree(partionTableTagsHash, NULL); xmlHashFree(partionTableTagsHash, NULL);
return 0; return 0;
} }

388
nominatim/index.c
View File

@@ -21,25 +21,25 @@ extern int verbose;
void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile) void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile)
{ {
struct index_thread_data * thread_data; struct index_thread_data * thread_data;
pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER;
int tuples, count, sleepcount; int tuples, count, sleepcount;
time_t rankStartTime; time_t rankStartTime;
int rankTotalTuples; int rankTotalTuples;
int rankCountTuples; int rankCountTuples;
float rankPerSecond; float rankPerSecond;
PGconn *conn; PGconn *conn;
PGresult * res; PGresult * res;
PGresult * resSectors; PGresult * resSectors;
PGresult * resPlaces; PGresult * resPlaces;
PGresult * resNULL; PGresult * resNULL;
int rank; int rank;
int i; int i;
int iSector; int iSector;
int iResult; int iResult;
const char *paramValues[2]; const char *paramValues[2];
int paramLengths[2]; int paramLengths[2];
@@ -49,20 +49,21 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
uint32_t sector; uint32_t sector;
xmlTextWriterPtr writer; xmlTextWriterPtr writer;
pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER;
Oid pg_prepare_params[2]; Oid pg_prepare_params[2];
conn = PQconnectdb(conninfo); conn = PQconnectdb(conninfo);
if (PQstatus(conn) != CONNECTION_OK) { if (PQstatus(conn) != CONNECTION_OK)
{
fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
pg_prepare_params[0] = PG_OID_INT4; pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(conn, "index_sectors", res = PQprepare(conn, "index_sectors",
"select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status > 0 group by geometry_sector order by geometry_sector", "select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status > 0 group by geometry_sector order by geometry_sector",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed preparing index_sectors: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed preparing index_sectors: %s\n", PQerrorMessage(conn));
@@ -72,8 +73,8 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
pg_prepare_params[0] = PG_OID_INT4; pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(conn, "index_nosectors", res = PQprepare(conn, "index_nosectors",
"select 0::integer,count(*) from placex where rank_search = $1 and indexed_status > 0", "select 0::integer,count(*) from placex where rank_search = $1 and indexed_status > 0",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed preparing index_sectors: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed preparing index_sectors: %s\n", PQerrorMessage(conn));
@@ -84,8 +85,8 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
pg_prepare_params[0] = PG_OID_INT4; pg_prepare_params[0] = PG_OID_INT4;
pg_prepare_params[1] = PG_OID_INT4; pg_prepare_params[1] = PG_OID_INT4;
res = PQprepare(conn, "index_sector_places", res = PQprepare(conn, "index_sector_places",
"select place_id from placex where rank_search = $1 and geometry_sector = $2 and indexed_status > 0", "select place_id from placex where rank_search = $1 and geometry_sector = $2 and indexed_status > 0",
2, pg_prepare_params); 2, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed preparing index_sector_places: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed preparing index_sector_places: %s\n", PQerrorMessage(conn));
@@ -95,8 +96,8 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
pg_prepare_params[0] = PG_OID_INT4; pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(conn, "index_nosector_places", res = PQprepare(conn, "index_nosector_places",
"select place_id from placex where rank_search = $1 and indexed_status > 0 order by geometry_sector", "select place_id from placex where rank_search = $1 and indexed_status > 0 order by geometry_sector",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed preparing index_nosector_places: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed preparing index_nosector_places: %s\n", PQerrorMessage(conn));
@@ -106,230 +107,231 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
// Build the data for each thread // Build the data for each thread
thread_data = (struct index_thread_data *)malloc(sizeof(struct index_thread_data)*num_threads); thread_data = (struct index_thread_data *)malloc(sizeof(struct index_thread_data)*num_threads);
for (i = 0; i < num_threads; i++) for (i = 0; i < num_threads; i++)
{ {
thread_data[i].conn = PQconnectdb(conninfo); thread_data[i].conn = PQconnectdb(conninfo);
if (PQstatus(thread_data[i].conn) != CONNECTION_OK) { if (PQstatus(thread_data[i].conn) != CONNECTION_OK)
fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(thread_data[i].conn)); {
exit(EXIT_FAILURE); fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(thread_data[i].conn));
} exit(EXIT_FAILURE);
}
pg_prepare_params[0] = PG_OID_INT4; pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(thread_data[i].conn, "index_placex", res = PQprepare(thread_data[i].conn, "index_placex",
"update placex set indexed_status = 0 where place_id = $1", "update placex set indexed_status = 0 where place_id = $1",
1, pg_prepare_params); 1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed preparing index_placex: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed preparing index_placex: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
res = PQexec(thread_data[i].conn, "set enable_seqscan = false"); res = PQexec(thread_data[i].conn, "set enable_seqscan = false");
if (PQresultStatus(res) != PGRES_COMMAND_OK) if (PQresultStatus(res) != PGRES_COMMAND_OK)
{ {
fprintf(stderr, "Failed disabling sequential scan: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Failed disabling sequential scan: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
nominatim_exportCreatePreparedQueries(thread_data[i].conn); nominatim_exportCreatePreparedQueries(thread_data[i].conn);
} }
// Create the output file // Create the output file
writer = NULL; writer = NULL;
if (structuredoutputfile) if (structuredoutputfile)
{ {
writer = nominatim_exportXMLStart(structuredoutputfile); writer = nominatim_exportXMLStart(structuredoutputfile);
} }
fprintf(stderr, "Starting indexing rank (%i to %i) using %i treads\n", rank_min, rank_max, num_threads); fprintf(stderr, "Starting indexing rank (%i to %i) using %i treads\n", rank_min, rank_max, num_threads);
for (rank = rank_min; rank <= rank_max; rank++) for (rank = rank_min; rank <= rank_max; rank++)
{ {
printf("Starting rank %d\n", rank); printf("Starting rank %d\n", rank);
rankCountTuples = 0; rankCountTuples = 0;
rankPerSecond = 0; rankPerSecond = 0;
paramRank = PGint32(rank); paramRank = PGint32(rank);
paramValues[0] = (char *)&paramRank; paramValues[0] = (char *)&paramRank;
paramLengths[0] = sizeof(paramRank); paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1; paramFormats[0] = 1;
if (rank < 16) if (rank < 16)
resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1); resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
else else
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1); resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK) if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resSectors); PQclear(resSectors);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (PQftype(resSectors, 0) != PG_OID_INT4) if (PQftype(resSectors, 0) != PG_OID_INT4)
{ {
fprintf(stderr, "Sector value has unexpected type\n"); fprintf(stderr, "Sector value has unexpected type\n");
PQclear(resSectors); PQclear(resSectors);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (PQftype(resSectors, 1) != PG_OID_INT8) if (PQftype(resSectors, 1) != PG_OID_INT8)
{ {
fprintf(stderr, "Sector value has unexpected type\n"); fprintf(stderr, "Sector value has unexpected type\n");
PQclear(resSectors); PQclear(resSectors);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
rankTotalTuples = 0; rankTotalTuples = 0;
for (iSector = 0; iSector < PQntuples(resSectors); iSector++) for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
{ {
rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))); rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)));
} }
rankStartTime = time(0); rankStartTime = time(0);
for (iSector = 0; iSector <= PQntuples(resSectors); iSector++) for (iSector = 0; iSector <= PQntuples(resSectors); iSector++)
{ {
if (iSector > 0) if (iSector > 0)
{ {
resPlaces = PQgetResult(conn); resPlaces = PQgetResult(conn);
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK) if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
{ {
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces); PQclear(resPlaces);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (PQftype(resPlaces, 0) != PG_OID_INT4) if (PQftype(resPlaces, 0) != PG_OID_INT4)
{ {
fprintf(stderr, "Place_id value has unexpected type\n"); fprintf(stderr, "Place_id value has unexpected type\n");
PQclear(resPlaces); PQclear(resPlaces);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
resNULL = PQgetResult(conn); resNULL = PQgetResult(conn);
if (resNULL != NULL) if (resNULL != NULL)
{ {
fprintf(stderr, "Unexpected non-null response\n"); fprintf(stderr, "Unexpected non-null response\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
if (iSector < PQntuples(resSectors)) if (iSector < PQntuples(resSectors))
{ {
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
//printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)))); //printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
// Get all the place_id's for this sector // Get all the place_id's for this sector
paramRank = PGint32(rank); paramRank = PGint32(rank);
paramValues[0] = (char *)&paramRank; paramValues[0] = (char *)&paramRank;
paramLengths[0] = sizeof(paramRank); paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1; paramFormats[0] = 1;
paramSector = PGint32(sector); paramSector = PGint32(sector);
paramValues[1] = (char *)&paramSector; paramValues[1] = (char *)&paramSector;
paramLengths[1] = sizeof(paramSector); paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1; paramFormats[1] = 1;
if (rank < 16) if (rank < 16)
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1); iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
else else
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1); iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
if (!iResult) if (!iResult)
{ {
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn)); fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces); PQclear(resPlaces);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
if (iSector > 0) if (iSector > 0)
{ {
count = 0; count = 0;
rankPerSecond = 0; rankPerSecond = 0;
tuples = PQntuples(resPlaces); tuples = PQntuples(resPlaces);
if (tuples > 0) if (tuples > 0)
{ {
// Spawn threads // Spawn threads
for (i = 0; i < num_threads; i++) for (i = 0; i < num_threads; i++)
{ {
thread_data[i].res = resPlaces; thread_data[i].res = resPlaces;
thread_data[i].tuples = tuples; thread_data[i].tuples = tuples;
thread_data[i].count = &count; thread_data[i].count = &count;
thread_data[i].count_mutex = &count_mutex; thread_data[i].count_mutex = &count_mutex;
thread_data[i].writer = writer; thread_data[i].writer = writer;
thread_data[i].writer_mutex = &writer_mutex; thread_data[i].writer_mutex = &writer_mutex;
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]); pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
} }
// Monitor threads to give user feedback // Monitor threads to give user feedback
sleepcount = 0; sleepcount = 0;
while(count < tuples) while (count < tuples)
{ {
usleep(1000); usleep(1000);
// Aim for one update per second // Aim for one update per second
if (sleepcount++ > 500) if (sleepcount++ > 500)
{ {
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1); rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
printf(" Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond); printf(" Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
sleepcount = 0; sleepcount = 0;
} }
} }
// Wait for everything to finish // Wait for everything to finish
for (i = 0; i < num_threads; i++) for (i = 0; i < num_threads; i++)
{ {
pthread_join(thread_data[i].thread, NULL); pthread_join(thread_data[i].thread, NULL);
} }
rankCountTuples += tuples; rankCountTuples += tuples;
} }
// Finished sector // Finished sector
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1); rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
printf(" Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond); printf(" Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
PQclear(resPlaces); PQclear(resPlaces);
} }
} }
// Finished rank // Finished rank
printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
PQclear(resSectors); PQclear(resSectors);
} }
if (writer) if (writer)
{ {
nominatim_exportXMLEnd(writer); nominatim_exportXMLEnd(writer);
} }
} }
void *nominatim_indexThread(void * thread_data_in) void *nominatim_indexThread(void * thread_data_in)
{ {
struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in; struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in;
PGresult *res; PGresult *res;
const char *paramValues[1]; const char *paramValues[1];
int paramLengths[1]; int paramLengths[1];
int paramFormats[1]; int paramFormats[1];
uint32_t paramPlaceID; uint32_t paramPlaceID;
uint32_t place_id; uint32_t place_id;
time_t updateStartTime; time_t updateStartTime;
while(1) while (1)
{ {
pthread_mutex_lock( thread_data->count_mutex ); pthread_mutex_lock( thread_data->count_mutex );
if (*(thread_data->count) >= thread_data->tuples) if (*(thread_data->count) >= thread_data->tuples)
{ {
pthread_mutex_unlock( thread_data->count_mutex ); pthread_mutex_unlock( thread_data->count_mutex );
break; break;
} }
place_id = PGint32(*((uint32_t *)PQgetvalue(thread_data->res, *thread_data->count, 0))); place_id = PGint32(*((uint32_t *)PQgetvalue(thread_data->res, *thread_data->count, 0)));
(*thread_data->count)++; (*thread_data->count)++;
pthread_mutex_unlock( thread_data->count_mutex ); pthread_mutex_unlock( thread_data->count_mutex );
if (verbose) printf(" Processing place_id %d\n", place_id); if (verbose) printf(" Processing place_id %d\n", place_id);
updateStartTime = time(0); updateStartTime = time(0);
paramPlaceID = PGint32(place_id); paramPlaceID = PGint32(place_id);
paramValues[0] = (char *)&paramPlaceID; paramValues[0] = (char *)&paramPlaceID;
paramLengths[0] = sizeof(paramPlaceID); paramLengths[0] = sizeof(paramPlaceID);
paramFormats[0] = 1; paramFormats[0] = 1;
@@ -341,13 +343,13 @@ void *nominatim_indexThread(void * thread_data_in)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
PQclear(res); PQclear(res);
if (difftime(time(0), updateStartTime) > 1) printf(" Slow place_id %d\n", place_id); if (difftime(time(0), updateStartTime) > 1) printf(" Slow place_id %d\n", place_id);
if (thread_data->writer) if (thread_data->writer)
{ {
nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex); nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex);
} }
} }
return NULL; return NULL;
} }

19
nominatim/index.h
View File

@@ -4,15 +4,16 @@
#include <libxml/encoding.h> #include <libxml/encoding.h>
#include <libxml/xmlwriter.h> #include <libxml/xmlwriter.h>
struct index_thread_data{ struct index_thread_data
pthread_t thread; {
PGconn * conn; pthread_t thread;
PGresult * res; PGconn * conn;
int tuples; PGresult * res;
int * count; int tuples;
pthread_mutex_t * count_mutex; int * count;
xmlTextWriterPtr writer; pthread_mutex_t * count_mutex;
pthread_mutex_t * writer_mutex; xmlTextWriterPtr writer;
pthread_mutex_t * writer_mutex;
}; };
void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile); void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile);
void *nominatim_indexThread(void * thread_data_in); void *nominatim_indexThread(void * thread_data_in);

206
nominatim/input.c
View File

@@ -19,44 +19,46 @@
#include "input.h" #include "input.h"
struct Input { struct Input
char *name; {
enum { plainFile, gzipFile, bzip2File } type; char *name;
void *fileHandle; enum { plainFile, gzipFile, bzip2File } type;
// needed by bzip2 when decompressing from multiple streams. other void *fileHandle;
// decompressors must ignore it. // needed by bzip2 when decompressing from multiple streams. other
FILE *systemHandle; // decompressors must ignore it.
int eof; FILE *systemHandle;
char buf[4096]; int eof;
int buf_ptr, buf_fill; char buf[4096];
int buf_ptr, buf_fill;
}; };
// tries to re-open the bz stream at the next stream start. // tries to re-open the bz stream at the next stream start.
// returns 0 on success, -1 on failure. // returns 0 on success, -1 on failure.
int bzReOpen(struct Input *ctx, int *error) { int bzReOpen(struct Input *ctx, int *error)
// for copying out the last unused part of the block which {
// has an EOS token in it. needed for re-initialising the // for copying out the last unused part of the block which
// next stream. // has an EOS token in it. needed for re-initialising the
unsigned char unused[BZ_MAX_UNUSED]; // next stream.
void *unused_tmp_ptr = NULL; unsigned char unused[BZ_MAX_UNUSED];
int nUnused, i; void *unused_tmp_ptr = NULL;
int nUnused, i;
BZ2_bzReadGetUnused(error, (BZFILE *)(ctx->fileHandle), &unused_tmp_ptr, &nUnused); BZ2_bzReadGetUnused(error, (BZFILE *)(ctx->fileHandle), &unused_tmp_ptr, &nUnused);
if (*error != BZ_OK) return -1; if (*error != BZ_OK) return -1;
// when bzReadClose is called the unused buffer is deallocated,
// so it needs to be copied somewhere safe first.
for (i = 0; i < nUnused; ++i)
unused[i] = ((unsigned char *)unused_tmp_ptr)[i];
BZ2_bzReadClose(error, (BZFILE *)(ctx->fileHandle));
if (*error != BZ_OK) return -1;
// reassign the file handle // when bzReadClose is called the unused buffer is deallocated,
ctx->fileHandle = BZ2_bzReadOpen(error, ctx->systemHandle, 0, 0, unused, nUnused); // so it needs to be copied somewhere safe first.
if (ctx->fileHandle == NULL || *error != BZ_OK) return -1; for (i = 0; i < nUnused; ++i)
unused[i] = ((unsigned char *)unused_tmp_ptr)[i];
return 0; BZ2_bzReadClose(error, (BZFILE *)(ctx->fileHandle));
if (*error != BZ_OK) return -1;
// reassign the file handle
ctx->fileHandle = BZ2_bzReadOpen(error, ctx->systemHandle, 0, 0, unused, nUnused);
if (ctx->fileHandle == NULL || *error != BZ_OK) return -1;
return 0;
} }
int readFile(void *context, char * buffer, int len) int readFile(void *context, char * buffer, int len)
@@ -67,36 +69,40 @@ int readFile(void *context, char * buffer, int len)
if (ctx->eof || (len == 0)) if (ctx->eof || (len == 0))
return 0; return 0;
switch(ctx->type) {
case plainFile:
l = read(*(int *)f, buffer, len);
if (l <= 0) ctx->eof = 1;
break;
case gzipFile:
l = gzread((gzFile)f, buffer, len);
if (l <= 0) ctx->eof = 1;
break;
case bzip2File:
l = BZ2_bzRead(&error, (BZFILE *)f, buffer, len);
// error codes BZ_OK and BZ_STREAM_END are both "OK", but the stream switch (ctx->type)
// end means the reader needs to be reset from the original handle. {
if (error != BZ_OK) { case plainFile:
// for stream errors, try re-opening the stream before admitting defeat. l = read(*(int *)f, buffer, len);
if (error != BZ_STREAM_END || bzReOpen(ctx, &error) != 0) { if (l <= 0) ctx->eof = 1;
l = 0; break;
ctx->eof = 1; case gzipFile:
} l = gzread((gzFile)f, buffer, len);
} if (l <= 0) ctx->eof = 1;
break; break;
default: case bzip2File:
fprintf(stderr, "Bad file type\n"); l = BZ2_bzRead(&error, (BZFILE *)f, buffer, len);
break;
// error codes BZ_OK and BZ_STREAM_END are both "OK", but the stream
// end means the reader needs to be reset from the original handle.
if (error != BZ_OK)
{
// for stream errors, try re-opening the stream before admitting defeat.
if (error != BZ_STREAM_END || bzReOpen(ctx, &error) != 0)
{
l = 0;
ctx->eof = 1;
}
}
break;
default:
fprintf(stderr, "Bad file type\n");
break;
} }
if (l < 0) { if (l < 0)
fprintf(stderr, "File reader received error %d (%d)\n", l, error); {
fprintf(stderr, "File reader received error %d (%d)\n", l, error);
l = 0; l = 0;
} }
@@ -107,12 +113,14 @@ char inputGetChar(void *context)
{ {
struct Input *ctx = context; struct Input *ctx = context;
if (ctx->buf_ptr == ctx->buf_fill) { if (ctx->buf_ptr == ctx->buf_fill)
{
ctx->buf_fill = readFile(context, &ctx->buf[0], sizeof(ctx->buf)); ctx->buf_fill = readFile(context, &ctx->buf[0], sizeof(ctx->buf));
ctx->buf_ptr = 0; ctx->buf_ptr = 0;
if (ctx->buf_fill == 0) if (ctx->buf_fill == 0)
return 0; return 0;
if (ctx->buf_fill < 0) { if (ctx->buf_fill < 0)
{
perror("Error while reading file"); perror("Error while reading file");
exit(1); exit(1);
} }
@@ -138,32 +146,43 @@ void *inputOpen(const char *name)
ctx->name = strdup(name); ctx->name = strdup(name);
if (ext && !strcmp(ext, ".gz")) { if (ext && !strcmp(ext, ".gz"))
{
ctx->fileHandle = (void *)gzopen(name, "rb"); ctx->fileHandle = (void *)gzopen(name, "rb");
ctx->type = gzipFile; ctx->type = gzipFile;
} else if (ext && !strcmp(ext, ".bz2")) { }
int error = 0; else if (ext && !strcmp(ext, ".bz2"))
ctx->systemHandle = fopen(name, "rb"); {
if (!ctx->systemHandle) { int error = 0;
fprintf(stderr, "error while opening file %s\n", name); ctx->systemHandle = fopen(name, "rb");
exit(10); if (!ctx->systemHandle)
} {
fprintf(stderr, "error while opening file %s\n", name);
exit(10);
}
ctx->fileHandle = (void *)BZ2_bzReadOpen(&error, ctx->systemHandle, 0, 0, NULL, 0); ctx->fileHandle = (void *)BZ2_bzReadOpen(&error, ctx->systemHandle, 0, 0, NULL, 0);
ctx->type = bzip2File; ctx->type = bzip2File;
} else { }
else
{
int *pfd = malloc(sizeof(pfd)); int *pfd = malloc(sizeof(pfd));
if (pfd) { if (pfd)
if (!strcmp(name, "-")) { {
if (!strcmp(name, "-"))
{
*pfd = STDIN_FILENO; *pfd = STDIN_FILENO;
} else { }
else
{
int flags = O_RDONLY; int flags = O_RDONLY;
#ifdef O_LARGEFILE #ifdef O_LARGEFILE
flags |= O_LARGEFILE; flags |= O_LARGEFILE;
#endif #endif
*pfd = open(name, flags); *pfd = open(name, flags);
if (*pfd < 0) { if (*pfd < 0)
{
free(pfd); free(pfd);
pfd = NULL; pfd = NULL;
} }
@@ -172,7 +191,8 @@ void *inputOpen(const char *name)
ctx->fileHandle = (void *)pfd; ctx->fileHandle = (void *)pfd;
ctx->type = plainFile; ctx->type = plainFile;
} }
if (!ctx->fileHandle) { if (!ctx->fileHandle)
{
fprintf(stderr, "error while opening file %s\n", name); fprintf(stderr, "error while opening file %s\n", name);
exit(10); exit(10);
} }
@@ -186,20 +206,21 @@ int inputClose(void *context)
struct Input *ctx = context; struct Input *ctx = context;
void *f = ctx->fileHandle; void *f = ctx->fileHandle;
switch(ctx->type) { switch (ctx->type)
case plainFile: {
close(*(int *)f); case plainFile:
free(f); close(*(int *)f);
break; free(f);
case gzipFile: break;
gzclose((gzFile)f); case gzipFile:
break; gzclose((gzFile)f);
case bzip2File: break;
BZ2_bzclose((BZFILE *)f); case bzip2File:
break; BZ2_bzclose((BZFILE *)f);
default: break;
fprintf(stderr, "Bad file type\n"); default:
break; fprintf(stderr, "Bad file type\n");
break;
} }
free(ctx->name); free(ctx->name);
@@ -211,7 +232,8 @@ xmlTextReaderPtr inputUTF8(const char *name)
{ {
void *ctx = inputOpen(name); void *ctx = inputOpen(name);
if (!ctx) { if (!ctx)
{
fprintf(stderr, "Input reader create failed for: %s\n", name); fprintf(stderr, "Input reader create failed for: %s\n", name);
return NULL; return NULL;
} }

98
nominatim/nominatim.c
View File

@@ -83,7 +83,8 @@ static void long_usage(char *arg0)
fprintf(stderr, " -v|--verbose\t\tVerbose output.\n"); fprintf(stderr, " -v|--verbose\t\tVerbose output.\n");
fprintf(stderr, "\n"); fprintf(stderr, "\n");
if (sizeof(int*) == 4) { if (sizeof(int*) == 4)
{
fprintf(stderr, "\n\nYou are running this on 32bit system - this will not work\n"); fprintf(stderr, "\n\nYou are running this on 32bit system - this will not work\n");
} }
} }
@@ -112,9 +113,11 @@ int main(int argc, char *argv[])
fprintf(stderr, "nominatim SVN version %s\n\n", VERSION); fprintf(stderr, "nominatim SVN version %s\n\n", VERSION);
while (1) { while (1)
{
int c, option_index = 0; int c, option_index = 0;
static struct option long_options[] = { static struct option long_options[] =
{
{"help", 0, 0, 'h'}, {"help", 0, 0, 'h'},
{"verbose", 0, 0, 'v'}, {"verbose", 0, 0, 'v'},
@@ -140,55 +143,86 @@ int main(int argc, char *argv[])
if (c == -1) if (c == -1)
break; break;
switch (c) { switch (c)
case 'v': verbose=1; break; {
case 'd': db=optarg; break; case 'v':
case 'U': username=optarg; break; verbose=1;
case 'W': pass_prompt=1; break; break;
case 'H': host=optarg; break; case 'd':
case 'P': port=optarg; break; db=optarg;
case 'h': long_usage_bool=1; break; break;
case 'i': index=1; break; case 'U':
case 'e': export=1; break; username=optarg;
case 'I': import=1; break; break;
case 't': threads=atoi(optarg); break; case 'W':
case 'F': file=optarg; break; pass_prompt=1;
case 'T': tagsfile=optarg; break; break;
case '?': case 'H':
default: host=optarg;
short_usage(argv[0]); break;
exit(EXIT_FAILURE); case 'P':
port=optarg;
break;
case 'h':
long_usage_bool=1;
break;
case 'i':
index=1;
break;
case 'e':
export=1;
break;
case 'I':
import=1;
break;
case 't':
threads=atoi(optarg);
break;
case 'F':
file=optarg;
break;
case 'T':
tagsfile=optarg;
break;
case '?':
default:
short_usage(argv[0]);
exit(EXIT_FAILURE);
} }
} }
if (long_usage_bool) { if (long_usage_bool)
{
long_usage(argv[0]); long_usage(argv[0]);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (threads < 1) threads = 1; if (threads < 1) threads = 1;
/* /*
if (argc == optind) { // No non-switch arguments if (argc == optind) { // No non-switch arguments
short_usage(argv[0]); short_usage(argv[0]);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
*/ */
if (index && import) { if (index && import)
{
fprintf(stderr, "Error: --index and --import options can not be used on the same database!\n"); fprintf(stderr, "Error: --index and --import options can not be used on the same database!\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (pass_prompt) if (pass_prompt)
password = simple_prompt("Password:", 100, 0); password = simple_prompt("Password:", 100, 0);
else { else
{
password = getenv("PGPASS"); password = getenv("PGPASS");
} }
// Test the database connection // Test the database connection
conninfo = build_conninfo(db, username, password, host, port); conninfo = build_conninfo(db, username, password, host, port);
conn = PQconnectdb(conninfo); conn = PQconnectdb(conninfo);
if (PQstatus(conn) != CONNECTION_OK) { if (PQstatus(conn) != CONNECTION_OK)
{
fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn)); fprintf(stderr, "Connection to database failed: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
@@ -196,7 +230,7 @@ int main(int argc, char *argv[])
if (!index && !export && !import) if (!index && !export && !import)
{ {
fprintf(stderr, "Please select index, export or import.\n"); fprintf(stderr, "Please select index, export or import.\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if (index) nominatim_index(0, 30, threads, conninfo, file); if (index) nominatim_index(0, 30, threads, conninfo, file);

37
nominatim/nominatim.h
View File

@@ -4,24 +4,25 @@
#define MAX(x,y) (x > y?x:y) #define MAX(x,y) (x > y?x:y)
#define MIN(x,y) (x < y?x:y) #define MIN(x,y) (x < y?x:y)
struct output_options { struct output_options
const char *conninfo; /* Connection info string */ {
const char *prefix; /* prefix for table names */ const char *conninfo; /* Connection info string */
int scale; /* scale for converting coordinates to fixed point */ const char *prefix; /* prefix for table names */
int projection; /* SRS of projection */ int scale; /* scale for converting coordinates to fixed point */
int append; /* Append to existing data */ int projection; /* SRS of projection */
int slim; /* In slim mode */ int append; /* Append to existing data */
int cache; /* Memory usable for cache in MB */ int slim; /* In slim mode */
struct middle_t *mid; /* Mid storage to use */ int cache; /* Memory usable for cache in MB */
const char *tblsindex; /* Pg Tablespace to store indexes */ struct middle_t *mid; /* Mid storage to use */
const char *style; /* style file to use */ const char *tblsindex; /* Pg Tablespace to store indexes */
int expire_tiles_zoom; /* Zoom level for tile expiry list */ const char *style; /* style file to use */
int expire_tiles_zoom_min; /* Minimum zoom level for tile expiry list */ int expire_tiles_zoom; /* Zoom level for tile expiry list */
const char *expire_tiles_filename; /* File name to output expired tiles list to */ int expire_tiles_zoom_min; /* Minimum zoom level for tile expiry list */
int enable_hstore; /* add an additional hstore column with objects key/value pairs */ const char *expire_tiles_filename; /* File name to output expired tiles list to */
int enable_multi; /* Output multi-geometries intead of several simple geometries */ int enable_hstore; /* add an additional hstore column with objects key/value pairs */
char** hstore_columns; /* list of columns that should be written into their own hstore column */ int enable_multi; /* Output multi-geometries intead of several simple geometries */
int n_hstore_columns; /* number of hstore columns */ char** hstore_columns; /* list of columns that should be written into their own hstore column */
int n_hstore_columns; /* number of hstore columns */
}; };
void exit_nicely(void); void exit_nicely(void);

12
nominatim/postgresql.c
View File

@@ -12,22 +12,26 @@ const char *build_conninfo(const char *db, const char *username, const char *pas
strcat(conninfo, db); strcat(conninfo, db);
strcat(conninfo, "'"); strcat(conninfo, "'");
if (username) { if (username)
{
strcat(conninfo, " user='"); strcat(conninfo, " user='");
strcat(conninfo, username); strcat(conninfo, username);
strcat(conninfo, "'"); strcat(conninfo, "'");
} }
if (password) { if (password)
{
strcat(conninfo, " password='"); strcat(conninfo, " password='");
strcat(conninfo, password); strcat(conninfo, password);
strcat(conninfo, "'"); strcat(conninfo, "'");
} }
if (host) { if (host)
{
strcat(conninfo, " host='"); strcat(conninfo, " host='");
strcat(conninfo, host); strcat(conninfo, host);
strcat(conninfo, "'"); strcat(conninfo, "'");
} }
if (port) { if (port)
{
strcat(conninfo, " port='"); strcat(conninfo, " port='");
strcat(conninfo, port); strcat(conninfo, port);
strcat(conninfo, "'"); strcat(conninfo, "'");

181
nominatim/sprompt.c
View File

@@ -11,7 +11,7 @@
* $PostgreSQL: pgsql/src/port/sprompt.c,v 1.18 2006/10/04 00:30:14 momjian Exp $ * $PostgreSQL: pgsql/src/port/sprompt.c,v 1.18 2006/10/04 00:30:14 momjian Exp $
* *
*------------------------------------------------------------------------- *-------------------------------------------------------------------------
* *
* PostgreSQL Database Management System * PostgreSQL Database Management System
* (formerly known as Postgres, then as Postgres95) * (formerly known as Postgres, then as Postgres95)
* *
@@ -76,124 +76,125 @@ extern char *simple_prompt(const char *prompt, int maxlen, int echo);
char * char *
simple_prompt(const char *prompt, int maxlen, int echo) simple_prompt(const char *prompt, int maxlen, int echo)
{ {
int length; int length;
char *destination; char *destination;
FILE *termin, FILE *termin,
*termout; *termout;
#ifdef HAVE_TERMIOS_H #ifdef HAVE_TERMIOS_H
struct termios t_orig, struct termios t_orig,
t; t;
#else #else
#ifdef WIN32 #ifdef WIN32
HANDLE t = NULL; HANDLE t = NULL;
LPDWORD t_orig = NULL; LPDWORD t_orig = NULL;
#endif #endif
#endif #endif
destination = (char *) malloc(maxlen + 1); destination = (char *) malloc(maxlen + 1);
if (!destination) if (!destination)
return NULL; return NULL;
/* /*
* Do not try to collapse these into one "w+" mode file. Doesn't work on * Do not try to collapse these into one "w+" mode file. Doesn't work on
* some platforms (eg, HPUX 10.20). * some platforms (eg, HPUX 10.20).
*/ */
termin = fopen(DEVTTY, "r"); termin = fopen(DEVTTY, "r");
termout = fopen(DEVTTY, "w"); termout = fopen(DEVTTY, "w");
if (!termin || !termout if (!termin || !termout
#ifdef WIN32 #ifdef WIN32
/* See DEVTTY comment for msys */ /* See DEVTTY comment for msys */
|| (getenv("OSTYPE") && strcmp(getenv("OSTYPE"), "msys") == 0) || (getenv("OSTYPE") && strcmp(getenv("OSTYPE"), "msys") == 0)
#endif #endif
) )
{ {
if (termin) if (termin)
fclose(termin); fclose(termin);
if (termout) if (termout)
fclose(termout); fclose(termout);
termin = stdin; termin = stdin;
termout = stderr; termout = stderr;
} }
#ifdef HAVE_TERMIOS_H #ifdef HAVE_TERMIOS_H
if (!echo) if (!echo)
{ {
tcgetattr(fileno(termin), &t); tcgetattr(fileno(termin), &t);
t_orig = t; t_orig = t;
t.c_lflag &= ~ECHO; t.c_lflag &= ~ECHO;
tcsetattr(fileno(termin), TCSAFLUSH, &t); tcsetattr(fileno(termin), TCSAFLUSH, &t);
} }
#else #else
#ifdef WIN32 #ifdef WIN32
if (!echo) if (!echo)
{ {
/* get a new handle to turn echo off */ /* get a new handle to turn echo off */
t_orig = (LPDWORD) malloc(sizeof(DWORD)); t_orig = (LPDWORD) malloc(sizeof(DWORD));
t = GetStdHandle(STD_INPUT_HANDLE); t = GetStdHandle(STD_INPUT_HANDLE);
/* save the old configuration first */ /* save the old configuration first */
GetConsoleMode(t, t_orig); GetConsoleMode(t, t_orig);
/* set to the new mode */ /* set to the new mode */
SetConsoleMode(t, ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT); SetConsoleMode(t, ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT);
} }
#endif #endif
#endif #endif
if (prompt) if (prompt)
{ {
fputs(prompt, termout); fputs(prompt, termout);
fflush(termout); fflush(termout);
} }
if (fgets(destination, maxlen + 1, termin) == NULL) if (fgets(destination, maxlen + 1, termin) == NULL)
destination[0] = '\0'; destination[0] = '\0';
length = strlen(destination); length = strlen(destination);
if (length > 0 && destination[length - 1] != '\n') if (length > 0 && destination[length - 1] != '\n')
{ {
/* eat rest of the line */ /* eat rest of the line */
char buf[128]; char buf[128];
int buflen; int buflen;
do do
{ {
if (fgets(buf, sizeof(buf), termin) == NULL) if (fgets(buf, sizeof(buf), termin) == NULL)
break; break;
buflen = strlen(buf); buflen = strlen(buf);
} while (buflen > 0 && buf[buflen - 1] != '\n'); }
} while (buflen > 0 && buf[buflen - 1] != '\n');
}
if (length > 0 && destination[length - 1] == '\n') if (length > 0 && destination[length - 1] == '\n')
/* remove trailing newline */ /* remove trailing newline */
destination[length - 1] = '\0'; destination[length - 1] = '\0';
#ifdef HAVE_TERMIOS_H #ifdef HAVE_TERMIOS_H
if (!echo) if (!echo)
{ {
tcsetattr(fileno(termin), TCSAFLUSH, &t_orig); tcsetattr(fileno(termin), TCSAFLUSH, &t_orig);
fputs("\n", termout); fputs("\n", termout);
fflush(termout); fflush(termout);
} }
#else #else
#ifdef WIN32 #ifdef WIN32
if (!echo) if (!echo)
{ {
/* reset to the original console mode */ /* reset to the original console mode */
SetConsoleMode(t, *t_orig); SetConsoleMode(t, *t_orig);
fputs("\n", termout); fputs("\n", termout);
fflush(termout); fflush(termout);
free(t_orig); free(t_orig);
} }
#endif #endif
#endif #endif
if (termin != stdin) if (termin != stdin)
{ {
fclose(termin); fclose(termin);
fclose(termout); fclose(termout);
} }
return destination; return destination;
} }