hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-07 10:34:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

add logging of broken polygons, improve address export, better diff output in export, bug in address calculation(#3522), reaply #3520, lost in migration

Browse Source
This commit is contained in:
Brian Quinion
2011-02-07 11:13:18 +00:00
parent bb9f0d81c0
commit 3ea1b35b32
8 changed files with 373 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
nominatim/export.c
View File

@@ -18,6 +18,8 @@
extern int verbose;
int mode = 0;
void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile)
{
xmlTextWriterPtr writer;
@@ -129,7 +131,7 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch
tuples = PQntuples(resPlaces);
for (i = 0; i < tuples; i++)
{
nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL);
nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL, NULL);
rankTotalDone++;
if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000);
}
@@ -150,7 +152,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_details",
"select osm_type, osm_id, class, type, name, housenumber, country_code, ST_AsText(geometry), admin_level, rank_address, rank_search from placex where place_id = $1",
"select placex.osm_type, placex.osm_id, placex.class, placex.type, placex.name, placex.housenumber, placex.country_code, ST_AsText(placex.geometry), placex.admin_level, placex.rank_address, placex.rank_search, placex.parent_place_id, parent.osm_type, parent.osm_id, placex.indexed_status from placex left outer join placex as parent on (placex.parent_place_id = parent.place_id) where placex.place_id = $1",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
@@ -224,23 +226,16 @@ xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile)
fprintf(stderr, "xmlTextWriterWriteAttribute failed\n");
exit(EXIT_FAILURE);
}
if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
{
fprintf(stderr, "xmlTextWriterStartElement failed\n");
exit(EXIT_FAILURE);
}
mode = 0;
return writer;
}
void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
{
// End <add>
if (xmlTextWriterEndElement(writer) < 0)
{
fprintf(stderr, "xmlTextWriterEndElement failed\n");
exit(EXIT_FAILURE);
}
nominatim_exportEndMode(writer);
// End <osmStructured>
if (xmlTextWriterEndElement(writer) < 0)
{
@@ -255,135 +250,237 @@ void nominatim_exportXMLEnd(xmlTextWriterPtr writer)
xmlFreeTextWriter(writer);
}
/*
* Requirements: the prepared queries must exist
*/
void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex)
void nominatim_exportStartMode(xmlTextWriterPtr writer, int newMode)
{
PGresult * res;
PGresult * resNames;
PGresult * resAddress;
PGresult * resExtraTags;
if (mode == newMode) return;
int i;
nominatim_exportEndMode(writer);
switch(newMode)
{
case 0:
break;
case 1:
if (xmlTextWriterStartElement(writer, BAD_CAST "add") < 0)
{
fprintf(stderr, "xmlTextWriterStartElement failed\n");
exit(EXIT_FAILURE);
}
break;
case 2:
if (xmlTextWriterStartElement(writer, BAD_CAST "update") < 0)
{
fprintf(stderr, "xmlTextWriterStartElement failed\n");
exit(EXIT_FAILURE);
}
break;
case 3:
if (xmlTextWriterStartElement(writer, BAD_CAST "delete") < 0)
{
fprintf(stderr, "xmlTextWriterStartElement failed\n");
exit(EXIT_FAILURE);
}
break;
}
mode = newMode;
}
void nominatim_exportEndMode(xmlTextWriterPtr writer)
{
if (!mode) return;
if (xmlTextWriterEndElement(writer) < 0)
{
fprintf(stderr, "xmlTextWriterEndElement failed\n");
exit(EXIT_FAILURE);
}
}
void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet)
{
const char * paramValues[1];
int paramLengths[1];
int paramFormats[1];
uint64_t paramPlaceID;
paramPlaceID = PGint64(place_id);
paramValues[0] = (char *)&paramPlaceID;
paramLengths[0] = sizeof(paramPlaceID);
paramFormats[0] = 1;
res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
querySet->res = PQexecPrepared(conn, "placex_details", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(querySet->res) != PGRES_TUPLES_OK)
{
fprintf(stderr, "placex_details: SELECT failed: %s", PQerrorMessage(conn));
PQclear(res);
PQclear(querySet->res);
exit(EXIT_FAILURE);
}
resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(resNames) != PGRES_TUPLES_OK)
querySet->resNames = PQexecPrepared(conn, "placex_names", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(querySet->resNames) != PGRES_TUPLES_OK)
{
fprintf(stderr, "placex_names: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resNames);
PQclear(querySet->resNames);
exit(EXIT_FAILURE);
}
resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(resAddress) != PGRES_TUPLES_OK)
querySet->resAddress = PQexecPrepared(conn, "placex_address", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(querySet->resAddress) != PGRES_TUPLES_OK)
{
fprintf(stderr, "placex_address: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resAddress);
PQclear(querySet->resAddress);
exit(EXIT_FAILURE);
}
resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(resExtraTags) != PGRES_TUPLES_OK)
querySet->resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(querySet->resExtraTags) != PGRES_TUPLES_OK)
{
fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resExtraTags);
PQclear(querySet->resExtraTags);
exit(EXIT_FAILURE);
}
}
if (writer_mutex) pthread_mutex_lock( writer_mutex );
void nominatim_exportFreeQueries(struct export_data * querySet)
{
PQclear(querySet->res);
PQclear(querySet->resNames);
PQclear(querySet->resAddress);
PQclear(querySet->resExtraTags);
}
/*
* Requirements: the prepared queries must exist
*/
void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet)
{
struct export_data querySet;
int i;
nominatim_exportPlaceQueries(place_id, conn, &querySet);
// Add, modify or delete?
if (prevQuerySet)
{
if ((PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "100") == 0) || PQntuples(querySet.res))
{
// Delete
if (writer_mutex) pthread_mutex_lock( writer_mutex );
nominatim_exportStartMode(writer, 3);
xmlTextWriterStartElement(writer, BAD_CAST "feature");
xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 0));
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(prevQuerySet->res, 0, 3));
xmlTextWriterEndElement(writer);
if (writer_mutex) pthread_mutex_unlock( writer_mutex );
nominatim_exportFreeQueries(&querySet);
return;
}
if (PQgetvalue(prevQuerySet->res, 0, 14) && strcmp(PQgetvalue(prevQuerySet->res, 0, 14), "1") == 0)
{
// Add
if (writer_mutex) pthread_mutex_lock( writer_mutex );
nominatim_exportStartMode(writer, 1);
}
else
{
// Update, but only if something has changed
// TODO: detect changes
if (writer_mutex) pthread_mutex_lock( writer_mutex );
nominatim_exportStartMode(writer, 2);
}
}
else
{
// Add
if (writer_mutex) pthread_mutex_lock( writer_mutex );
nominatim_exportStartMode(writer, 1);
}
xmlTextWriterStartElement(writer, BAD_CAST "feature");
xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "place_id", "%li", place_id);
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(res, 0, 0));
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(res, 0, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(res, 0, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(res, 0, 3));
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(res, 0, 9));
xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(res, 0, 10));
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.res, 0, 0));
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.res, 0, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.res, 0, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.res, 0, 3));
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.res, 0, 9));
xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(querySet.res, 0, 10));
xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_place_id", BAD_CAST PQgetvalue(querySet.res, 0, 11));
xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_type", BAD_CAST PQgetvalue(querySet.res, 0, 12));
xmlTextWriterWriteAttribute(writer, BAD_CAST "parent_id", BAD_CAST PQgetvalue(querySet.res, 0, 13));
if (PQntuples(resNames))
if (PQntuples(querySet.resNames))
{
xmlTextWriterStartElement(writer, BAD_CAST "names");
for (i = 0; i < PQntuples(resNames); i++)
for (i = 0; i < PQntuples(querySet.resNames); i++)
{
xmlTextWriterStartElement(writer, BAD_CAST "name");
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resNames, i, 0));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resNames, i, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resNames, i, 0));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resNames, i, 1));
xmlTextWriterEndElement(writer);
}
xmlTextWriterEndElement(writer);
}
if (PQgetvalue(res, 0, 5) && strlen(PQgetvalue(res, 0, 5)))
if (PQgetvalue(querySet.res, 0, 5) && strlen(PQgetvalue(querySet.res, 0, 5)))
{
xmlTextWriterStartElement(writer, BAD_CAST "houseNumber");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 5));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 5));
xmlTextWriterEndElement(writer);
}
if (PQgetvalue(res, 0, 8) && strlen(PQgetvalue(res, 0, 8)))
if (PQgetvalue(querySet.res, 0, 8) && strlen(PQgetvalue(querySet.res, 0, 8)))
{
xmlTextWriterStartElement(writer, BAD_CAST "adminLevel");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 8));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 8));
xmlTextWriterEndElement(writer);
}
if (PQgetvalue(res, 0, 6) && strlen(PQgetvalue(res, 0, 6)))
if (PQgetvalue(querySet.res, 0, 6) && strlen(PQgetvalue(querySet.res, 0, 6)))
{
xmlTextWriterStartElement(writer, BAD_CAST "countryCode");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 6));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 6));
xmlTextWriterEndElement(writer);
}
if (PQntuples(resAddress) > 0)
if (PQntuples(querySet.resAddress) > 0)
{
xmlTextWriterStartElement(writer, BAD_CAST "address");
for (i = 0; i < PQntuples(resAddress); i++)
for (i = 0; i < PQntuples(querySet.resAddress); i++)
{
xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(resAddress, i, 5))));
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(resAddress, i, 5));
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resAddress, i, 0));
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(resAddress, i, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(resAddress, i, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(resAddress, i, 3));
xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(resAddress, i, 4));
xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(resAddress, i, 6));
xmlTextWriterStartElement(writer, BAD_CAST getRankLabel(atoi(PQgetvalue(querySet.resAddress, i, 5))));
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(querySet.resAddress, i, 5));
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resAddress, i, 0));
xmlTextWriterWriteAttribute(writer, BAD_CAST "id", BAD_CAST PQgetvalue(querySet.resAddress, i, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(querySet.resAddress, i, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(querySet.resAddress, i, 3));
xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(querySet.resAddress, i, 4));
xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(querySet.resAddress, i, 6));
xmlTextWriterEndElement(writer);
}
xmlTextWriterEndElement(writer);
}
if (PQntuples(resExtraTags))
if (PQntuples(querySet.resExtraTags))
{
xmlTextWriterStartElement(writer, BAD_CAST "tags");
for (i = 0; i < PQntuples(resExtraTags); i++)
for (i = 0; i < PQntuples(querySet.resExtraTags); i++)
{
xmlTextWriterStartElement(writer, BAD_CAST "tag");
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resExtraTags, i, 0));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resExtraTags, i, 1));
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(querySet.resExtraTags, i, 0));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.resExtraTags, i, 1));
xmlTextWriterEndElement(writer);
}
@@ -392,17 +489,14 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 7));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(querySet.res, 0, 7));
xmlTextWriterEndElement(writer);
xmlTextWriterEndElement(writer); // </feature>
if (writer_mutex) pthread_mutex_unlock( writer_mutex );
PQclear(res);
PQclear(resNames);
PQclear(resAddress);
PQclear(resExtraTags);
nominatim_exportFreeQueries(&querySet);
}
const char * getRankLabel(int rank)

18
nominatim/export.h
View File

@@ -5,11 +5,27 @@
#include <libxml/xmlwriter.h>
#include <stdint.h>
struct export_data
{
PGresult * res;
PGresult * resNames;
PGresult * resAddress;
PGresult * resExtraTags;
};
void nominatim_export(int rank_min, int rank_max, const char *conninfo, const char *structuredoutputfile);
void nominatim_exportCreatePreparedQueries(PGconn * conn);
xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile);
void nominatim_exportXMLEnd(xmlTextWriterPtr writer);
void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex);
void nominatim_exportEndMode(xmlTextWriterPtr writer);
void nominatim_exportPlaceQueries(uint64_t place_id, PGconn * conn, struct export_data * querySet);
void nominatim_exportFreeQueries(struct export_data * querySet);
void nominatim_exportPlace(uint64_t place_id, PGconn * conn,
xmlTextWriterPtr writer, pthread_mutex_t * writer_mutex, struct export_data * prevQuerySet);
const char * getRankLabel(int rank);
#endif

100
nominatim/import.c
View File

@@ -50,6 +50,9 @@ struct feature
xmlChar * rankAddress;
xmlChar * rankSearch;
xmlChar * countryCode;
xmlChar * parentPlaceID;
xmlChar * parentType;
xmlChar * parentID;
xmlChar * adminLevel;
xmlChar * houseNumber;
xmlChar * geometry;
@@ -67,6 +70,7 @@ int featureNameLines = 0;
int featureExtraTagLines = 0;
int featureCount = 0;
xmlHashTablePtr partionTableTagsHash;
xmlHashTablePtr partionTableTagsHashDelete;
char featureNameString[MAX_FEATURENAMESTRING];
char featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
@@ -135,13 +139,17 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
feature.rankAddress = xmlTextReaderGetAttribute(reader, BAD_CAST "rank");
feature.rankSearch = xmlTextReaderGetAttribute(reader, BAD_CAST "importance");
feature.parentPlaceID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_place_id");
feature.parentType = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_type");
feature.parentID = xmlTextReaderGetAttribute(reader, BAD_CAST "parent_id");
feature.countryCode = NULL;
feature.adminLevel = NULL;
feature.houseNumber = NULL;
feature.geometry = NULL;
featureAddressLines = 0;
featureNameLines = 0;
featureExtraTagLines = 0;
featureExtraTagLines = 0;
return;
}
@@ -312,7 +320,6 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
{
PGresult * res;
PGresult * resPlaceID;
const char * paramValues[11];
char * place_id;
char * partionQueryName;
@@ -348,9 +355,9 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
}
}
*/
place_id = feature.placeID;
place_id = (char *)feature.placeID;
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE)
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_DELETE || fileMode == FILEMODE_ADD)
{
paramValues[0] = (const char *)place_id;
res = PQexecPrepared(conn, "placex_delete", 1, paramValues, NULL, NULL, 0);
@@ -379,6 +386,19 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
exit(EXIT_FAILURE);
}
PQclear(res);
partionQueryName = xmlHashLookup2(partionTableTagsHashDelete, feature.key, feature.value);
if (partionQueryName)
{
res = PQexecPrepared(conn, partionQueryName, 1, paramValues, NULL, NULL, 0);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "%s: DELETE failed: %s", partionQueryName, PQerrorMessage(conn));
PQclear(res);
exit(EXIT_FAILURE);
}
PQclear(res);
}
}
if (fileMode == FILEMODE_UPDATE || fileMode == FILEMODE_ADD)
@@ -398,8 +418,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
lineValueLen = 0;
for (i = 0; i < featureNameLines; i++)
{
lineTypeLen = strlen(BAD_CAST featureName[i].type);
lineValueLen = strlen(BAD_CAST featureName[i].value);
lineTypeLen = (int)strlen((char *) featureName[i].type);
lineValueLen = (int)strlen((char *) featureName[i].value);
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
{
fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
@@ -407,11 +427,11 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
}
if (namePos) strcpy(featureNameString+(namePos++), ",");
strcpy(featureNameString+(namePos++), "\"");
strcpy(featureNameString+namePos, BAD_CAST featureName[i].type);
strcpy(featureNameString+namePos, (char*) featureName[i].type);
namePos += lineTypeLen;
strcpy(featureNameString+namePos, "\"=>\"");
namePos += 4;
strcpy(featureNameString+namePos, BAD_CAST featureName[i].value);
strcpy(featureNameString+namePos, (char *) featureName[i].value);
namePos += lineValueLen;
strcpy(featureNameString+(namePos++), "\"");
}
@@ -426,8 +446,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
lineValueLen = 0;
for (i = 0; i < featureExtraTagLines; i++)
{
lineTypeLen = strlen(BAD_CAST featureExtraTag[i].type);
lineValueLen = strlen(BAD_CAST featureExtraTag[i].value);
lineTypeLen = strlen((char *) featureExtraTag[i].type);
lineValueLen = strlen((char *) featureExtraTag[i].value);
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
{
fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
@@ -435,30 +455,36 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
}
if (namePos) strcpy(featureExtraTagString+(namePos++),",");
strcpy(featureExtraTagString+(namePos++), "\"");
strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].type);
strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].type);
namePos += lineTypeLen;
strcpy(featureExtraTagString+namePos, "\"=>\"");
namePos += 4;
strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].value);
strcpy(featureExtraTagString+namePos, (char *) featureExtraTag[i].value);
namePos += lineValueLen;
strcpy(featureExtraTagString+(namePos++), "\"");
}
}
paramValues[6] = (const char *)featureExtraTagString;
paramValues[7] = (const char *)feature.adminLevel;
paramValues[8] = (const char *)feature.houseNumber;
paramValues[9] = (const char *)feature.rankAddress;
paramValues[10] = (const char *)feature.rankSearch;
paramValues[11] = (const char *)feature.geometry;
res = PQexecPrepared(conn, "placex_insert", 12, paramValues, NULL, NULL, 0);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
paramValues[7] = (const char *)feature.parentPlaceID;
paramValues[8] = (const char *)feature.adminLevel;
paramValues[9] = (const char *)feature.houseNumber;
paramValues[10] = (const char *)feature.rankAddress;
paramValues[11] = (const char *)feature.rankSearch;
paramValues[12] = (const char *)feature.geometry;
if (strlen(paramValues[3]))
{
fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
PQclear(res);
exit(EXIT_FAILURE);
res = PQexecPrepared(conn, "placex_insert", 13, paramValues, NULL, NULL, 0);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
fprintf(stderr, "index_placex: INSERT failed: %s %s %s", paramValues[0], paramValues[1], paramValues[2]);
PQclear(res);
exit(EXIT_FAILURE);
}
PQclear(res);
}
PQclear(res);
for (i = 0; i < featureAddressLines; i++)
{
@@ -513,7 +539,6 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
exit(EXIT_FAILURE);
}
PQclear(res);
}
}
@@ -595,6 +620,7 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
}
partionTableTagsHash = xmlHashCreate(200);
partionTableTagsHashDelete = xmlHashCreate(200);
partionTagsFile = fopen(partionTagsFilename, "rt");
if (!partionTagsFile)
@@ -636,6 +662,27 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
}
xmlHashAddEntry2(partionTableTagsHash, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
partionQueryName = malloc(strlen("partition_delete_")+strlen(osmkey)+strlen(osmvalue)+2);
strcpy(partionQueryName, "partition_delete_");
strcat(partionQueryName, osmkey);
strcat(partionQueryName, "_");
strcat(partionQueryName, osmvalue);
strcpy(partionQuerySQL, "delete from place_classtype_");
strcat(partionQuerySQL, osmkey);
strcat(partionQuerySQL, "_");
strcat(partionQuerySQL, osmvalue);
strcat(partionQuerySQL, " where place_id = $1::integer");
res = PQprepare(conn, partionQueryName, partionQuerySQL, 1, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed to prepare %s: %s\n", partionQueryName, PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
xmlHashAddEntry2(partionTableTagsHashDelete, BAD_CAST osmkey, BAD_CAST osmvalue, BAD_CAST partionQueryName);
}
res = PQprepare(conn, "get_new_place_id",
@@ -657,8 +704,8 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
}
res = PQprepare(conn, "placex_insert",
"insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,admin_level,housenumber,rank_address,rank_search,geometry) "
"values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, st_setsrid($12, 4326))",
"insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,parent_place_id,admin_level,housenumber,rank_address,rank_search,geometry) "
"values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, st_setsrid($13, 4326))",
12, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
@@ -740,6 +787,7 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
xmlFreeTextReader(reader);
xmlHashFree(partionTableTagsHash, NULL);
xmlHashFree(partionTableTagsHashDelete, NULL);
return 0;
}

25
nominatim/index.c
View File

@@ -40,6 +40,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
int i;
int iSector;
int iResult;
int bSkip;
const char *paramValues[2];
int paramLengths[2];
@@ -160,7 +161,8 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
// if (rank < 16)
// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
// else
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
{
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
@@ -215,7 +217,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
if (iSector < PQntuples(resSectors))
{
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
//printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
// printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
// Get all the place_id's for this sector
paramRank = PGint32(rank);
@@ -226,9 +228,9 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
paramValues[1] = (char *)&paramSector;
paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1;
if (rankTotalTuples-rankCountTuples < num_threads*20)
if (rankTotalTuples-rankCountTuples < num_threads*1000)
{
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
}
else
{
@@ -292,7 +294,10 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
PQclear(resPlaces);
}
if (rankTotalTuples-rankCountTuples < num_threads*20) iSector = PQntuples(resSectors);
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
{
iSector = PQntuples(resSectors) - 1;
}
}
// Finished rank
printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
@@ -309,6 +314,7 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
void *nominatim_indexThread(void * thread_data_in)
{
struct index_thread_data * thread_data = (struct index_thread_data * )thread_data_in;
struct export_data querySet;
PGresult *res;
@@ -339,6 +345,12 @@ void *nominatim_indexThread(void * thread_data_in)
int done = 0;
while(!done)
{
if (thread_data->writer)
{
nominatim_exportPlaceQueries(place_id, thread_data->conn, &querySet);
}
paramPlaceID = PGint32(place_id);
paramValues[0] = (char *)&paramPlaceID;
paramLengths[0] = sizeof(paramPlaceID);
@@ -366,7 +378,8 @@ void *nominatim_indexThread(void * thread_data_in)
if (thread_data->writer)
{
nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex);
nominatim_exportPlace(place_id, thread_data->conn, thread_data->writer, thread_data->writer_mutex, &querySet);
nominatim_exportFreeQueries(&querySet);
}
}