hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-14 18:37:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

prot load-data function to python

Browse Source
This commit is contained in:
Sarah Hoffmann
2021-02-25 21:32:40 +01:00
parent 3c186f8030
commit 57db5819ef
4 changed files with 101 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
lib-php/setup/SetupClass.php
View File

@@ -119,133 +119,6 @@ class SetupFunctions
$this->pgsqlRunPartitionScript($sTemplate);
}
public function loadData($bDisableTokenPrecalc)
{
info('Drop old Data');
$oDB = $this->db();
$oDB->exec('TRUNCATE word');
echo '.';
$oDB->exec('TRUNCATE placex');
echo '.';
$oDB->exec('TRUNCATE location_property_osmline');
echo '.';
$oDB->exec('TRUNCATE place_addressline');
echo '.';
$oDB->exec('TRUNCATE location_area');
echo '.';
if (!$this->dbReverseOnly()) {
$oDB->exec('TRUNCATE search_name');
echo '.';
}
$oDB->exec('TRUNCATE search_name_blank');
echo '.';
$oDB->exec('DROP SEQUENCE seq_place');
echo '.';
$oDB->exec('CREATE SEQUENCE seq_place start 100000');
echo '.';
$sSQL = 'select distinct partition from country_name';
$aPartitions = $oDB->getCol($sSQL);
if (!$this->bNoPartitions) $aPartitions[] = 0;
foreach ($aPartitions as $sPartition) {
$oDB->exec('TRUNCATE location_road_'.$sPartition);
echo '.';
}
// used by getorcreate_word_id to ignore frequent partial words
$sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
$sSQL .= '$$ SELECT '.getSetting('MAX_WORD_FREQUENCY').' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
$oDB->exec($sSQL);
echo ".\n";
// pre-create the word list
if (!$bDisableTokenPrecalc) {
info('Loading word list');
$this->pgsqlRunScriptFile(CONST_DataDir.'/words.sql');
}
info('Load Data');
$sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry';
$aDBInstances = array();
$iLoadThreads = max(1, $this->iInstances - 1);
for ($i = 0; $i < $iLoadThreads; $i++) {
// https://secure.php.net/manual/en/function.pg-connect.php
$DSN = getSetting('DATABASE_DSN');
$DSN = preg_replace('/^pgsql:/', '', $DSN);
$DSN = preg_replace('/;/', ' ', $DSN);
$aDBInstances[$i] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW);
pg_ping($aDBInstances[$i]);
}
for ($i = 0; $i < $iLoadThreads; $i++) {
$sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
$sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
$sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')";
$sSQL .= ' and ST_IsValid(geometry)';
if ($this->bVerbose) echo "$sSQL\n";
if (!pg_send_query($aDBInstances[$i], $sSQL)) {
fail(pg_last_error($aDBInstances[$i]));
}
}
// last thread for interpolation lines
// https://secure.php.net/manual/en/function.pg-connect.php
$DSN = getSetting('DATABASE_DSN');
$DSN = preg_replace('/^pgsql:/', '', $DSN);
$DSN = preg_replace('/;/', ' ', $DSN);
$aDBInstances[$iLoadThreads] = pg_connect($DSN, PGSQL_CONNECT_FORCE_NEW);
pg_ping($aDBInstances[$iLoadThreads]);
$sSQL = 'insert into location_property_osmline';
$sSQL .= ' (osm_id, address, linegeo)';
$sSQL .= ' SELECT osm_id, address, geometry from place where ';
$sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";
if ($this->bVerbose) echo "$sSQL\n";
if (!pg_send_query($aDBInstances[$iLoadThreads], $sSQL)) {
fail(pg_last_error($aDBInstances[$iLoadThreads]));
}
$bFailed = false;
for ($i = 0; $i <= $iLoadThreads; $i++) {
while (($hPGresult = pg_get_result($aDBInstances[$i])) !== false) {
$resultStatus = pg_result_status($hPGresult);
// PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK,
// PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE,
// PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR
// echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n";
if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) {
$resultError = pg_result_error($hPGresult);
echo '-- error text ' . $i . ': ' . $resultError . "\n";
$bFailed = true;
}
}
}
if ($bFailed) {
fail('SQL errors loading placex and/or location_property_osmline tables');
}
for ($i = 0; $i < $this->iInstances; $i++) {
pg_close($aDBInstances[$i]);
}
echo "\n";
info('Reanalysing database');
$this->pgsqlRunScript('ANALYSE');
$sDatabaseDate = getDatabaseDate($oDB);
$oDB->exec('TRUNCATE import_status');
if (!$sDatabaseDate) {
warn('could not determine database date.');
} else {
$sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
$oDB->exec($sSQL);
echo "Latest data imported from $sDatabaseDate.\n";
}
}
public function importTigerData($sTigerPath)
{
info('Import Tiger data');