hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-26 11:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

postcode/zipcode improvements, finish work on handling extratags

Browse Source
This commit is contained in:
Brian Quinion
2010-12-07 13:41:02 +00:00
parent 491f5961e1
commit 631e8d09ab
11 changed files with 378 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
nominatim/export.c
View File

@@ -51,7 +51,7 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch
pg_prepare_params[0] = PG_OID_INT4;
res = PQprepare(conn, "index_sectors",
"select geometry_sector,count(*) from placex where rank_search = $1 and indexed = true group by geometry_sector order by geometry_sector",
"select geometry_sector,count(*) from placex where rank_search = $1 and indexed_status = 0 group by geometry_sector order by geometry_sector",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK) exit(EXIT_FAILURE);
PQclear(res);
@@ -118,7 +118,7 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
if (PQftype(resPlaces, 0) != PG_OID_INT8)
if (PQftype(resPlaces, 0) != PG_OID_INT4)
{
fprintf(stderr, "Place_id value has unexpected type\n");
PQclear(resPlaces);
@@ -128,7 +128,7 @@ void nominatim_export(int rank_min, int rank_max, const char *conninfo, const ch
tuples = PQntuples(resPlaces);
for(i = 0; i < tuples; i++)
{
nominatim_exportPlace(PGint64(*((uint64_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL);
nominatim_exportPlace(PGint32(*((uint32_t *)PQgetvalue(resPlaces, i, 0))), conn, writer, NULL);
rankTotalDone++;
if (rankTotalDone%1000 == 0) printf("Done %i (k)\n", rankTotalDone/1000);
}
@@ -160,7 +160,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_address",
"select osm_type,osm_id,class,type,distance,cached_rank_address from place_addressline join placex on (address_place_id = placex.place_id) where isaddress and place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc",
"select osm_type,osm_id,class,type,distance,cached_rank_address,isaddress from place_addressline join placex on (address_place_id = placex.place_id) where place_addressline.place_id = $1 and address_place_id != place_addressline.place_id order by cached_rank_address asc",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
@@ -171,7 +171,7 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_names",
"select (each(name)).key,(each(name)).value from (select name as name from placex where place_id = $1) as x",
"select (each(name)).key,(each(name)).value from (select name from placex where place_id = $1) as x",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
@@ -179,6 +179,17 @@ void nominatim_exportCreatePreparedQueries(PGconn * conn)
exit(EXIT_FAILURE);
}
PQclear(res);
pg_prepare_params[0] = PG_OID_INT8;
res = PQprepare(conn, "placex_extratags",
"select (each(extratags)).key,(each(extratags)).value from (select extratags from placex where place_id = $1) as x",
1, pg_prepare_params);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Error preparing placex_extratags: %s", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
}
xmlTextWriterPtr nominatim_exportXMLStart(const char *structuredoutputfile)
@@ -251,6 +262,7 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
PGresult * res;
PGresult * resNames;
PGresult * resAddress;
PGresult * resExtraTags;
int i;
@@ -289,6 +301,14 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
exit(EXIT_FAILURE);
}
resExtraTags = PQexecPrepared(conn, "placex_extratags", 1, paramValues, paramLengths, paramFormats, 0);
if (PQresultStatus(resExtraTags) != PGRES_TUPLES_OK)
{
fprintf(stderr, "placex_extratags: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resExtraTags);
exit(EXIT_FAILURE);
}
if (writer_mutex) pthread_mutex_lock( writer_mutex );
xmlTextWriterStartElement(writer, BAD_CAST "feature");
@@ -300,7 +320,7 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
xmlTextWriterWriteAttribute(writer, BAD_CAST "rank", BAD_CAST PQgetvalue(res, 0, 9));
xmlTextWriterWriteAttribute(writer, BAD_CAST "importance", BAD_CAST PQgetvalue(res, 0, 10));
if (PQgetvalue(res, 0, 4) && strlen(PQgetvalue(res, 0, 4)))
if (PQntuples(resNames))
{
xmlTextWriterStartElement(writer, BAD_CAST "names");
@@ -348,11 +368,28 @@ void nominatim_exportPlace(uint64_t place_id, PGconn * conn, xmlTextWriterPtr wr
xmlTextWriterWriteAttribute(writer, BAD_CAST "key", BAD_CAST PQgetvalue(resAddress, i, 2));
xmlTextWriterWriteAttribute(writer, BAD_CAST "value", BAD_CAST PQgetvalue(resAddress, i, 3));
xmlTextWriterWriteAttribute(writer, BAD_CAST "distance", BAD_CAST PQgetvalue(resAddress, i, 4));
xmlTextWriterWriteAttribute(writer, BAD_CAST "isaddress", BAD_CAST PQgetvalue(resAddress, i, 6));
xmlTextWriterEndElement(writer);
}
xmlTextWriterEndElement(writer);
}
if (PQntuples(resExtraTags))
{
xmlTextWriterStartElement(writer, BAD_CAST "tags");
for(i = 0; i < PQntuples(resExtraTags); i++)
{
xmlTextWriterStartElement(writer, BAD_CAST "tag");
xmlTextWriterWriteAttribute(writer, BAD_CAST "type", BAD_CAST PQgetvalue(resExtraTags, i, 0));
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(resExtraTags, i, 1));
xmlTextWriterEndElement(writer);
}
xmlTextWriterEndElement(writer);
}
xmlTextWriterStartElement(writer, BAD_CAST "osmGeometry");
xmlTextWriterWriteString(writer, BAD_CAST PQgetvalue(res, 0, 7));
xmlTextWriterEndElement(writer);

124
nominatim/import.c
View File

@@ -18,10 +18,14 @@ typedef enum { FILEMODE_NONE, FILEMODE_ADD, FILEMODE_UPDATE, FILEMODE_DELETE } f
#define MAX_FEATUREADDRESS 500
#define MAX_FEATURENAMES 1000
#define MAX_FEATUREEXTRATAGS 100
#define MAX_FEATURENAMESTRING 100000
#define MAX_FEATUREEXTRATAGSTRING 50000
struct feature_address {
int place_id;
int rankAddress;
char isAddress[2];
xmlChar * type;
xmlChar * id;
xmlChar * key;
@@ -29,7 +33,7 @@ struct feature_address {
xmlChar * distance;
};
struct feature_name {
struct feature_tag {
xmlChar * type;
xmlChar * value;
};
@@ -52,14 +56,16 @@ int fileType = FILETYPE_NONE;
int fileMode = FILEMODE_ADD;
PGconn * conn;
struct feature_address featureAddress[MAX_FEATUREADDRESS];
struct feature_name featureName[MAX_FEATURENAMES];
struct feature_tag featureName[MAX_FEATURENAMES];
struct feature_tag featureExtraTag[MAX_FEATUREEXTRATAGS];
struct feature feature;
int featureAddressLines = 0;
int featureNameLines = 0;
int featureExtraTagLines = 0;
int featureCount = 0;
xmlHashTablePtr partionTableTagsHash;
char featureNameString[MAX_FEATURENAMESTRING];
char featureExtraTagString[MAX_FEATUREEXTRATAGSTRING];
void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
{
@@ -147,6 +153,19 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
}
return;
}
if (xmlStrEqual(name, BAD_CAST "tags")) return;
if (xmlStrEqual(name, BAD_CAST "tag"))
{
featureExtraTag[featureExtraTagLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureExtraTag[featureExtraTagLines].value = xmlTextReaderReadString(reader);
featureExtraTagLines++;
if (featureExtraTagLines >= MAX_FEATUREEXTRATAGS)
{
fprintf( stderr, "Too many extra tag elements\n");
exit_nicely();
}
return;
}
if (xmlStrEqual(name, BAD_CAST "osmGeometry"))
{
feature.geometry = xmlTextReaderReadString(reader);
@@ -248,6 +267,16 @@ void StartElement(xmlTextReaderPtr reader, const xmlChar *name)
featureAddress[featureAddressLines].rankAddress = atoi(value);
xmlFree(value);
value = (char*)xmlTextReaderGetAttribute(reader, BAD_CAST "isaddress");
if (!value)
{
fprintf( stderr, "Address element missing rank\n");
exit_nicely();
}
if (*value == 't') strcpy(featureAddress[featureAddressLines].isAddress, "t");
else strcpy(featureAddress[featureAddressLines].isAddress, "f");
xmlFree(value);
featureAddress[featureAddressLines].type = xmlTextReaderGetAttribute(reader, BAD_CAST "type");
featureAddress[featureAddressLines].id = xmlTextReaderGetAttribute(reader, BAD_CAST "id");
featureAddress[featureAddressLines].key = xmlTextReaderGetAttribute(reader, BAD_CAST "key");
@@ -273,7 +302,7 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
const char * paramValues[11];
char * place_id;
char * partionQueryName;
int i;
int i, namePos, lineTypeLen, lineValueLen;
if (xmlStrEqual(name, BAD_CAST "feature"))
{
@@ -345,13 +374,69 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
paramValues[2] = (const char *)feature.id;
paramValues[3] = (const char *)feature.key;
paramValues[4] = (const char *)feature.value;
// paramValues[5] = (const char *)feature.name;
paramValues[6] = (const char *)feature.adminLevel;
paramValues[7] = (const char *)feature.houseNumber;
paramValues[8] = (const char *)feature.rankAddress;
paramValues[9] = (const char *)feature.rankSearch;
paramValues[10] = (const char *)feature.geometry;
res = PQexecPrepared(conn, "placex_insert", 11, paramValues, NULL, NULL, 0);
featureNameString[0] = 0;
if (featureNameLines)
{
namePos = 0;
lineTypeLen = 0;
lineValueLen = 0;
for(i = 0; i < featureNameLines; i++)
{
lineTypeLen = strlen(BAD_CAST featureName[i].type);
lineValueLen = strlen(BAD_CAST featureName[i].value);
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATURENAMESTRING)
{
fprintf(stderr, "feature name too long: %s", (const char *)featureName[i].value);
break;
}
if (namePos) strcpy(featureNameString+(namePos++), ",");
strcpy(featureNameString+(namePos++), "\"");
strcpy(featureNameString+namePos, BAD_CAST featureName[i].type);
namePos += lineTypeLen;
strcpy(featureNameString+namePos, "\"=>\"");
namePos += 4;
strcpy(featureNameString+namePos, BAD_CAST featureName[i].value);
namePos += lineValueLen;
strcpy(featureNameString+(namePos++), "\"");
}
}
paramValues[5] = (const char *)featureNameString;
featureExtraTagString[0] = 0;
if (featureExtraTagLines)
{
namePos = 0;
lineTypeLen = 0;
lineValueLen = 0;
for(i = 0; i < featureExtraTagLines; i++)
{
lineTypeLen = strlen(BAD_CAST featureExtraTag[i].type);
lineValueLen = strlen(BAD_CAST featureExtraTag[i].value);
if (namePos+lineTypeLen+lineValueLen+7 > MAX_FEATUREEXTRATAGSTRING)
{
fprintf(stderr, "feature extra tag too long: %s", (const char *)featureExtraTag[i].value);
break;
}
if (namePos) strcpy(featureExtraTagString+(namePos++),",");
strcpy(featureExtraTagString+(namePos++), "\"");
strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].type);
namePos += lineTypeLen;
strcpy(featureExtraTagString+namePos, "\"=>\"");
namePos += 4;
strcpy(featureExtraTagString+namePos, BAD_CAST featureExtraTag[i].value);
namePos += lineValueLen;
strcpy(featureExtraTagString+(namePos++), "\"");
}
}
paramValues[6] = (const char *)featureExtraTagString;
paramValues[7] = (const char *)feature.adminLevel;
paramValues[8] = (const char *)feature.houseNumber;
paramValues[9] = (const char *)feature.rankAddress;
paramValues[10] = (const char *)feature.rankSearch;
paramValues[11] = (const char *)feature.geometry;
res = PQexecPrepared(conn, "placex_insert", 12, paramValues, NULL, NULL, 0);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "index_placex: INSERT failed: %s", PQerrorMessage(conn));
@@ -369,7 +454,8 @@ void EndElement(xmlTextReaderPtr reader, const xmlChar *name)
paramValues[3] = (const char *)featureAddress[i].id;
paramValues[4] = (const char *)featureAddress[i].key;
paramValues[5] = (const char *)featureAddress[i].value;
res = PQexecPrepared(conn, "place_addressline_insert", 6, paramValues, NULL, NULL, 0);
paramValues[6] = (const char *)featureAddress[i].isAddress;
res = PQexecPrepared(conn, "place_addressline_insert", 7, paramValues, NULL, NULL, 0);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "place_addressline_insert: INSERT failed: %s", PQerrorMessage(conn));
@@ -555,9 +641,9 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
}
res = PQprepare(conn, "placex_insert",
"insert into placex (place_id,osm_type,osm_id,class,type,name,admin_level,housenumber,rank_address,rank_search,geometry) "
"values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, st_setsrid($11, 4326))",
11, NULL);
"insert into placex (place_id,osm_type,osm_id,class,type,name,extratags,admin_level,housenumber,rank_address,rank_search,geometry) "
"values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, st_setsrid($12, 4326))",
12, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
@@ -573,14 +659,14 @@ int nominatim_import(const char *conninfo, const char *partionTagsFilename, cons
1, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed to prepare placex_insert: %s\n", PQerrorMessage(conn));
fprintf(stderr, "Failed to prepare search_name_insert: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
res = PQprepare(conn, "place_addressline_insert",
"insert into place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) "
"select $1, place_id, false, true, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
6, NULL);
"select $1, place_id, false, $7, $2, rank_address from placex where osm_type = $3 and osm_id = $4 and class = $5 and type = $6",
7, NULL);
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed to prepare place_addressline_insert: %s\n", PQerrorMessage(conn));

8
nominatim/index.c
View File

@@ -121,6 +121,14 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
fprintf(stderr, "Failed preparing index_placex: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
res = PQexec(thread_data[i].conn, "set enable_seqscan = false");
if (PQresultStatus(res) != PGRES_COMMAND_OK)
{
fprintf(stderr, "Failed disabling sequential scan: %s\n", PQerrorMessage(conn));
exit(EXIT_FAILURE);
}
PQclear(res);
nominatim_exportCreatePreparedQueries(thread_data[i].conn);