mirror of
https://github.com/osm-search/Nominatim.git
synced 2026-02-26 11:08:13 +00:00
index on geometry of interpolation lines, and more improvements.
This commit is contained in:
Markus Gail
@@ -21,34 +21,243 @@
|
||||
|
||||
extern int verbose;
|
||||
|
||||
void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile)
|
||||
void run_indexing(int rank, int interpolation, PGconn *conn, int num_threads,
|
||||
struct index_thread_data * thread_data, const char *structuredoutputfile)
|
||||
{
|
||||
struct index_thread_data * thread_data;
|
||||
pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
int tuples, count, sleepcount;
|
||||
|
||||
pthread_mutex_t count_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
time_t rankStartTime;
|
||||
int rankTotalTuples;
|
||||
int rankCountTuples;
|
||||
float rankPerSecond;
|
||||
|
||||
PGconn *conn;
|
||||
PGresult * res;
|
||||
|
||||
PGresult * resSectors;
|
||||
PGresult * resPlaces;
|
||||
PGresult * resNULL;
|
||||
|
||||
int rank;
|
||||
|
||||
int i;
|
||||
int iSector;
|
||||
int iResult;
|
||||
|
||||
|
||||
const char *paramValues[2];
|
||||
int paramLengths[2];
|
||||
int paramFormats[2];
|
||||
uint32_t paramRank;
|
||||
uint32_t paramSector;
|
||||
uint32_t sector;
|
||||
|
||||
xmlTextWriterPtr writer;
|
||||
pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
// Create the output file
|
||||
writer = NULL;
|
||||
if (structuredoutputfile)
|
||||
{
|
||||
writer = nominatim_exportXMLStart(structuredoutputfile);
|
||||
}
|
||||
|
||||
if (interpolation)
|
||||
{
|
||||
fprintf(stderr, "Starting interpolation lines (location_property_osmline)\n");
|
||||
}
|
||||
else
|
||||
{
|
||||
fprintf(stderr, "Starting rank %d\n", rank);
|
||||
}
|
||||
|
||||
rankCountTuples = 0;
|
||||
rankPerSecond = 0;
|
||||
|
||||
paramRank = PGint32(rank);
|
||||
paramValues[0] = (char *)¶mRank;
|
||||
paramLengths[0] = sizeof(paramRank);
|
||||
paramFormats[0] = 1;
|
||||
|
||||
if (interpolation)
|
||||
{
|
||||
resSectors = PQexecPrepared(conn, "index_sectors_osmline", 0, NULL, 0, NULL, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 0) != PG_OID_INT4)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 1) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
rankTotalTuples = 0;
|
||||
for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
|
||||
{
|
||||
rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)));
|
||||
}
|
||||
|
||||
rankStartTime = time(0);
|
||||
for (iSector = 0; iSector <= PQntuples(resSectors); iSector++)
|
||||
{
|
||||
if (iSector > 0)
|
||||
{
|
||||
resPlaces = PQgetResult(conn);
|
||||
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resPlaces, 0) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Place_id value has unexpected type\n");
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
resNULL = PQgetResult(conn);
|
||||
if (resNULL != NULL)
|
||||
{
|
||||
fprintf(stderr, "Unexpected non-null response\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (iSector < PQntuples(resSectors))
|
||||
{
|
||||
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
|
||||
// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
|
||||
|
||||
// Get all the place_id's for this sector
|
||||
paramRank = PGint32(rank);
|
||||
paramValues[0] = (char *)¶mRank;
|
||||
paramLengths[0] = sizeof(paramRank);
|
||||
paramFormats[0] = 1;
|
||||
paramSector = PGint32(sector);
|
||||
paramValues[1] = (char *)¶mSector;
|
||||
paramLengths[1] = sizeof(paramSector);
|
||||
paramFormats[1] = 1;
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*1000)
|
||||
{
|
||||
// no sectors
|
||||
if (interpolation)
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_nosector_places_osmline", 0, NULL, 0, NULL, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (interpolation)
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_sector_places_osmline", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
}
|
||||
if (!iResult)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
if (iSector > 0)
|
||||
{
|
||||
count = 0;
|
||||
rankPerSecond = 0;
|
||||
tuples = PQntuples(resPlaces);
|
||||
|
||||
if (tuples > 0)
|
||||
{
|
||||
// Spawn threads
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
thread_data[i].res = resPlaces;
|
||||
thread_data[i].tuples = tuples;
|
||||
thread_data[i].count = &count;
|
||||
thread_data[i].count_mutex = &count_mutex;
|
||||
thread_data[i].writer = writer;
|
||||
thread_data[i].writer_mutex = &writer_mutex;
|
||||
if (interpolation)
|
||||
{
|
||||
thread_data[i].table = 0; // use interpolations table
|
||||
}
|
||||
else
|
||||
{
|
||||
thread_data[i].table = 1; // use placex table
|
||||
}
|
||||
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
|
||||
}
|
||||
|
||||
// Monitor threads to give user feedback
|
||||
sleepcount = 0;
|
||||
while (count < tuples)
|
||||
{
|
||||
usleep(1000);
|
||||
|
||||
// Aim for one update per second
|
||||
if (sleepcount++ > 500)
|
||||
{
|
||||
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
|
||||
sleepcount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for everything to finish
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
pthread_join(thread_data[i].thread, NULL);
|
||||
}
|
||||
|
||||
rankCountTuples += tuples;
|
||||
}
|
||||
|
||||
// Finished sector
|
||||
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
|
||||
|
||||
PQclear(resPlaces);
|
||||
}
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
|
||||
{
|
||||
iSector = PQntuples(resSectors) - 1;
|
||||
}
|
||||
}
|
||||
// Finished rank
|
||||
fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED\n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
|
||||
|
||||
PQclear(resSectors);
|
||||
|
||||
|
||||
}
|
||||
|
||||
void nominatim_index(int rank_min, int rank_max, int num_threads, const char *conninfo, const char *structuredoutputfile)
|
||||
{
|
||||
struct index_thread_data * thread_data;
|
||||
|
||||
PGconn *conn;
|
||||
PGresult * res;
|
||||
|
||||
int rank;
|
||||
|
||||
int i;
|
||||
|
||||
xmlTextWriterPtr writer;
|
||||
pthread_mutex_t writer_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
@@ -182,324 +391,22 @@ void nominatim_index(int rank_min, int rank_max, int num_threads, const char *co
|
||||
nominatim_exportCreatePreparedQueries(thread_data[i].conn);
|
||||
}
|
||||
|
||||
// Create the output file
|
||||
writer = NULL;
|
||||
if (structuredoutputfile)
|
||||
{
|
||||
writer = nominatim_exportXMLStart(structuredoutputfile);
|
||||
}
|
||||
|
||||
fprintf(stderr, "Starting indexing rank (%i to %i) using %i threads\n", rank_min, rank_max, num_threads);
|
||||
|
||||
// first for the placex table
|
||||
for (rank = rank_min; rank <= rank_max; rank++)
|
||||
{
|
||||
// OSMLINE: do reindexing (=> reparenting) for interpolation lines at rank 30, but before all other objects of rank 30
|
||||
// reason: houses (rank 30) depend on the updated interpolation line, when reparenting (see placex_update in functions.sql)
|
||||
if (rank == 30)
|
||||
{
|
||||
fprintf(stderr, "Starting indexing interpolation lines (location_property_osmline)\n");
|
||||
rankCountTuples = 0;
|
||||
rankTotalTuples = 0;
|
||||
resSectors = PQexecPrepared(conn, "index_sectors_osmline", 0, NULL, 0, NULL, 1);
|
||||
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sectors_osmline: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 0) != PG_OID_INT4)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 1) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
rankStartTime = time(0);
|
||||
for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
|
||||
{
|
||||
rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)));
|
||||
}
|
||||
// do it only if tuples with indexed_status > 0 were found in osmline
|
||||
int nTuples = PQntuples(resSectors);
|
||||
if (nTuples > 0)
|
||||
{
|
||||
for (iSector = 0; iSector <= nTuples; iSector++)
|
||||
{
|
||||
if (iSector > 0)
|
||||
{
|
||||
resPlaces = PQgetResult(conn);
|
||||
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places: SELECT failed: %s\n", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resPlaces, 0) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Place_id value has unexpected type\n");
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
resNULL = PQgetResult(conn);
|
||||
if (resNULL != NULL)
|
||||
{
|
||||
fprintf(stderr, "Unexpected non-null response\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (iSector < nTuples)
|
||||
{
|
||||
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
|
||||
// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
|
||||
|
||||
// Get all the place_id's for this sector
|
||||
paramSector = PGint32(sector);
|
||||
paramValues[0] = (char *)¶mSector;
|
||||
paramLengths[0] = sizeof(paramSector);
|
||||
paramFormats[0] = 1;
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*1000)
|
||||
{
|
||||
// no sectors
|
||||
iResult = PQsendQueryPrepared(conn, "index_nosector_places_osmline", 0, NULL, 0, NULL, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_sector_places_osmline", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
if (!iResult)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places_osmline: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
if (iSector > 0)
|
||||
{
|
||||
count = 0;
|
||||
rankPerSecond = 0;
|
||||
tuples = PQntuples(resPlaces);
|
||||
|
||||
if (tuples > 0)
|
||||
{
|
||||
// Spawn threads
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
thread_data[i].res = resPlaces;
|
||||
thread_data[i].tuples = tuples;
|
||||
thread_data[i].count = &count;
|
||||
thread_data[i].count_mutex = &count_mutex;
|
||||
thread_data[i].writer = writer;
|
||||
thread_data[i].writer_mutex = &writer_mutex;
|
||||
thread_data[i].table = 0; // use osmline table
|
||||
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
|
||||
}
|
||||
// Monitor threads to give user feedback
|
||||
sleepcount = 0;
|
||||
while (count < tuples)
|
||||
{
|
||||
usleep(1000);
|
||||
|
||||
// Aim for one update per second
|
||||
if (sleepcount++ > 500)
|
||||
{
|
||||
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - Interpolation Lines ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - (rankCountTuples + count)))/(float)rankPerSecond);
|
||||
sleepcount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for everything to finish
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
pthread_join(thread_data[i].thread, NULL);
|
||||
}
|
||||
rankCountTuples += tuples;
|
||||
}
|
||||
// Finished sector
|
||||
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
|
||||
PQclear(resPlaces);
|
||||
}
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < nTuples)
|
||||
{
|
||||
iSector = nTuples - 1;
|
||||
}
|
||||
}
|
||||
PQclear(resSectors);
|
||||
}
|
||||
// Finished rank
|
||||
fprintf(stderr, "\r Done %i tuples in %i seconds- FINISHED\n", rankCountTuples,(int)(difftime(time(0), rankStartTime)));
|
||||
if (writer)
|
||||
{
|
||||
nominatim_exportXMLEnd(writer);
|
||||
}
|
||||
run_indexing(rank, 1, conn, num_threads, thread_data, structuredoutputfile);
|
||||
}
|
||||
fprintf(stderr, "Starting rank %d\n", rank);
|
||||
rankCountTuples = 0;
|
||||
rankPerSecond = 0;
|
||||
|
||||
paramRank = PGint32(rank);
|
||||
paramValues[0] = (char *)¶mRank;
|
||||
paramLengths[0] = sizeof(paramRank);
|
||||
paramFormats[0] = 1;
|
||||
// if (rank < 16)
|
||||
// resSectors = PQexecPrepared(conn, "index_nosectors", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
// else
|
||||
resSectors = PQexecPrepared(conn, "index_sectors", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
|
||||
if (PQresultStatus(resSectors) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sectors: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 0) != PG_OID_INT4)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resSectors, 1) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Sector value has unexpected type\n");
|
||||
PQclear(resSectors);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
rankTotalTuples = 0;
|
||||
for (iSector = 0; iSector < PQntuples(resSectors); iSector++)
|
||||
{
|
||||
rankTotalTuples += PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1)));
|
||||
}
|
||||
|
||||
rankStartTime = time(0);
|
||||
|
||||
for (iSector = 0; iSector <= PQntuples(resSectors); iSector++)
|
||||
{
|
||||
if (iSector > 0)
|
||||
{
|
||||
resPlaces = PQgetResult(conn);
|
||||
if (PQresultStatus(resPlaces) != PGRES_TUPLES_OK)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (PQftype(resPlaces, 0) != PG_OID_INT8)
|
||||
{
|
||||
fprintf(stderr, "Place_id value has unexpected type\n");
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
resNULL = PQgetResult(conn);
|
||||
if (resNULL != NULL)
|
||||
{
|
||||
fprintf(stderr, "Unexpected non-null response\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (iSector < PQntuples(resSectors))
|
||||
{
|
||||
sector = PGint32(*((uint32_t *)PQgetvalue(resSectors, iSector, 0)));
|
||||
// fprintf(stderr, "\n Starting sector %d size %ld\n", sector, PGint64(*((uint64_t *)PQgetvalue(resSectors, iSector, 1))));
|
||||
|
||||
// Get all the place_id's for this sector
|
||||
paramRank = PGint32(rank);
|
||||
paramValues[0] = (char *)¶mRank;
|
||||
paramLengths[0] = sizeof(paramRank);
|
||||
paramFormats[0] = 1;
|
||||
paramSector = PGint32(sector);
|
||||
paramValues[1] = (char *)¶mSector;
|
||||
paramLengths[1] = sizeof(paramSector);
|
||||
paramFormats[1] = 1;
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*1000)
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_nosector_places", 1, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
iResult = PQsendQueryPrepared(conn, "index_sector_places", 2, paramValues, paramLengths, paramFormats, 1);
|
||||
}
|
||||
if (!iResult)
|
||||
{
|
||||
fprintf(stderr, "index_sector_places: SELECT failed: %s", PQerrorMessage(conn));
|
||||
PQclear(resPlaces);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
if (iSector > 0)
|
||||
{
|
||||
count = 0;
|
||||
rankPerSecond = 0;
|
||||
tuples = PQntuples(resPlaces);
|
||||
|
||||
if (tuples > 0)
|
||||
{
|
||||
// Spawn threads
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
thread_data[i].res = resPlaces;
|
||||
thread_data[i].tuples = tuples;
|
||||
thread_data[i].count = &count;
|
||||
thread_data[i].count_mutex = &count_mutex;
|
||||
thread_data[i].writer = writer;
|
||||
thread_data[i].writer_mutex = &writer_mutex;
|
||||
thread_data[i].table = 1; // use placex table
|
||||
pthread_create(&thread_data[i].thread, NULL, &nominatim_indexThread, (void *)&thread_data[i]);
|
||||
}
|
||||
|
||||
// Monitor threads to give user feedback
|
||||
sleepcount = 0;
|
||||
while (count < tuples)
|
||||
{
|
||||
usleep(1000);
|
||||
|
||||
// Aim for one update per second
|
||||
if (sleepcount++ > 500)
|
||||
{
|
||||
rankPerSecond = ((float)rankCountTuples + (float)count) / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - Rank %i ETA (seconds): %f\n", (rankCountTuples + count), (int)(difftime(time(0), rankStartTime)), rankPerSecond, rank, ((float)(rankTotalTuples - (rankCountTuples + count)))/rankPerSecond);
|
||||
sleepcount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for everything to finish
|
||||
for (i = 0; i < num_threads; i++)
|
||||
{
|
||||
pthread_join(thread_data[i].thread, NULL);
|
||||
}
|
||||
|
||||
rankCountTuples += tuples;
|
||||
}
|
||||
|
||||
// Finished sector
|
||||
rankPerSecond = (float)rankCountTuples / MAX(difftime(time(0), rankStartTime),1);
|
||||
fprintf(stderr, " Done %i in %i @ %f per second - ETA (seconds): %f\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond, ((float)(rankTotalTuples - rankCountTuples))/rankPerSecond);
|
||||
|
||||
PQclear(resPlaces);
|
||||
}
|
||||
if (rankTotalTuples-rankCountTuples < num_threads*20 && iSector < PQntuples(resSectors))
|
||||
{
|
||||
iSector = PQntuples(resSectors) - 1;
|
||||
}
|
||||
}
|
||||
// Finished rank
|
||||
fprintf(stderr, "\r Done %i in %i @ %f per second - FINISHED \n\n", rankCountTuples, (int)(difftime(time(0), rankStartTime)), rankPerSecond);
|
||||
|
||||
PQclear(resSectors);
|
||||
run_indexing(rank, 0, conn, num_threads, thread_data, structuredoutputfile);
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (rank_max == 30)
|
||||
if (rank == 30)
|
||||
{
|
||||
// Close all connections
|
||||
for (i = 0; i < num_threads; i++)
|
||||
|
||||
Reference in New Issue
Block a user