hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-07 10:34:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

be a bit efficient about queuing up the places to be indexed

Browse Source
This commit is contained in:
Brian Quinion
2010-12-10 16:09:43 +00:00
parent 8193757427
commit 1427753846
1 changed files with 88 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
nominatim/index.c
View File

@@ -34,10 +34,12 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
PGresult * res; PGresult * res;
PGresult * resSectors; PGresult * resSectors;
PGresult * resPlaces; PGresult * resPlaces;
PGresult * resNULL;
int rank; int rank;
int i; int i;
int iSector; int iSector;
int iResult;
const char *paramValues[2]; const char *paramValues[2];
int paramLengths[2]; int paramLengths[2];
@@ -183,85 +185,107 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
} }
rankStartTime = time(0); rankStartTime = time(0);
for (iSector = 0; iSector < PQntuples(resSectors); iSector++) for (iSector = 0; iSector <= PQntuples(resSectors); iSector++)
{ {
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0))); if (iSector > 0)
//printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
// Get all the place_id's for this sector
paramRank = PGint32(rank);
paramValues[0] = (char *)&paramRank;
paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1;
paramSector = PGint32(sector);
paramValues[1] = (char *)&paramSector;
paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1;
if (rank < 16)
resPlaces = PQexecPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
else
resPlaces = PQexecPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
{
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
if (PQftype(resPlaces, 0) != PG_OID_INT4)
{ {
fprintf(stderr, "Place_id value has unexpected type\n"); resPlaces = PQgetResult(conn);
PQclear(resPlaces); if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
exit(EXIT_FAILURE); {
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
if (PQftype(resPlaces, 0) != PG_OID_INT4)
{
fprintf(stderr, "Place_id value has unexpected type\n");
PQclear(resPlaces);
exit(EXIT_FAILURE);
}
resNULL = PQgetResult(conn);
if (resNULL != NULL)
{
fprintf(stderr, "Unexpected non-null response\n");
exit(EXIT_FAILURE);
}
} }
count = 0; if (iSector < PQntuples(resSectors))
rankPerSecond = 0; {
tuples = PQntuples(resPlaces); sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
//printf("\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
if (tuples > 0) // Get all the place_id's for this sector
{ paramRank = PGint32(rank);
// Spawn threads paramValues[0] = (char *)&paramRank;
for (i = 0; i < num_threads; i++) paramLengths[0] = sizeof(paramRank);
paramFormats[0] = 1;
paramSector = PGint32(sector);
paramValues[1] = (char *)&paramSector;
paramLengths[1] = sizeof(paramSector);
paramFormats[1] = 1;
if (rank < 16)
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
else
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
if (!iResult)
{ {
thread_data[i].res = resPlaces; fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
thread_data[i].tuples = tuples; PQclear(resPlaces);
thread_data[i].count = &count; exit(EXIT_FAILURE);
thread_data[i].count_mutex = &count_mutex;
thread_data[i].writer = writer;
thread_data[i].writer_mutex = &writer_mutex;
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
} }
}
// Monitor threads to give user feedback if (iSector > 0)
sleepcount = 0; {
while(count < tuples) count = 0;
{ rankPerSecond = 0;
usleep(1000); tuples = PQntuples(resPlaces);
// Aim for one update per second if (tuples > 0)
if (sleepcount++ > 500) {
// Spawn threads
for (i = 0; i < num_threads; i++)
{ {
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1); thread_data[i].res = resPlaces;
printf(" Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond); thread_data[i].tuples = tuples;
sleepcount = 0; thread_data[i].count = &count;
thread_data[i].count_mutex = &count_mutex;
thread_data[i].writer = writer;
thread_data[i].writer_mutex = &writer_mutex;
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
} }
// Monitor threads to give user feedback
sleepcount = 0;
while(count < tuples)
{
usleep(1000);
// Aim for one update per second
if (sleepcount++ > 500)
{
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
printf(" Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
sleepcount = 0;
}
}
// Wait for everything to finish
for (i = 0; i < num_threads; i++)
{
pthread_join(thread_data[i].thread, NULL);
}
rankCountTuples += tuples;
} }
// Wait for everything to finish // Finished sector
for (i = 0; i < num_threads; i++) rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
{ printf(" Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
pthread_join(thread_data[i].thread, NULL);
}
rankCountTuples += tuples; PQclear(resPlaces);
} }
// Finished sector
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
printf(" Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
PQclear(resPlaces);
} }
// Finished rank // Finished rank
printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond); printf("\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);