safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

add a function for the intial indexing run

Browse Source 
Also moves postcodes to fully parallel indexing.
This commit is contained in:
Sarah Hoffmann
2021-02-25 11:25:01 +01:00
parent db5e78c879
commit 3c186f8030
7 changed files with 184 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
lib-php/admin/setup.php
View File

@@ -64,6 +64,16 @@ if ($aCMDResult['verbose']) {
$oNominatimCmd->addParams('--verbose');
}
// by default, use all but one processor, but never more than 15.
var_dump($aCMDResult);
$iInstances = max(1, $aCMDResult['threads'] ?? (min(16, getProcessorCount()) - 1));
function run($oCmd) {
global $iInstances;
$oCmd->addParams('--threads', $iInstances);
$oCmd->run(true);
}
//*******************************************************
// Making some sanity check:
@@ -81,7 +91,7 @@ $oSetup = new SetupFunctions($aCMDResult);
// go through complete process if 'all' is selected or start selected functions
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
$bDidSomething = true;
(clone($oNominatimCmd))->addParams('transition', '--create-db')->run(true);
run((clone($oNominatimCmd))->addParams('transition', '--create-db'));
}
if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
@@ -92,7 +102,7 @@ if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
$oCmd->addParams('--no-partitions');
}
$oCmd->run(true);
run($oCmd);
}
if ($aCMDResult['import-data'] || $aCMDResult['all']) {
@@ -104,7 +114,7 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) {
$oCmd->addParams('--drop');
}
$oCmd->run(true);
run($oCmd);
}
if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
@@ -131,6 +141,7 @@ if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
$bDidSomething = true;
// ignore errors!
(clone($oNominatimCmd))->addParams('refresh', '--wiki-data')->run();
}
@@ -152,12 +163,17 @@ if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
if ($aCMDResult['index'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->index($aCMDResult['index-noanalyse']);
$oCmd = (clone($oNominatimCmd))->addParams('transition', '--index');
if ($aCMDResult['index-noanalyse'] ?? false) {
$oCmd->addParams('--no-analyse');
}
run($oCmd);
}
if ($aCMDResult['drop']) {
$bDidSomething = true;
(clone($oNominatimCmd))->addParams('freeze')->run(true);
run((clone($oNominatimCmd))->addParams('freeze'));
}
if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
@@ -172,7 +188,7 @@ if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
if ($aCMDResult['setup-website'] || $aCMDResult['all']) {
$bDidSomething = true;
(clone($oNominatimCmd))->addParams('refresh', '--website')->run(true);
run((clone($oNominatimCmd))->addParams('refresh', '--website'));
}
// ******************************************************

44
lib-php/setup/SetupClass.php
View File

@@ -71,7 +71,6 @@ class SetupFunctions
if ($this->bVerbose) {
$this->oNominatimCmd->addParams('--verbose');
}
$this->oNominatimCmd->addParams('--threads', $this->iInstances);
}
public function createFunctions()
@@ -380,49 +379,6 @@ class SetupFunctions
$this->db()->exec($sSQL);
}
public function index($bIndexNoanalyse)
{
$this->checkModulePresence(); // raises exception on failure
$oBaseCmd = (clone $this->oNominatimCmd)->addParams('index');
info('Index ranks 0 - 4');
$oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4);
$iStatus = $oCmd->run();
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
info('Index administrative boundaries');
$oCmd = (clone $oBaseCmd)->addParams('--boundaries-only');
$iStatus = $oCmd->run();
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
info('Index ranks 5 - 25');
$oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 5, '--maxrank', 25);
$iStatus = $oCmd->run();
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
info('Index ranks 26 - 30');
$oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 26);
$iStatus = $oCmd->run();
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
info('Index postcodes');
$sSQL = 'UPDATE location_postcode SET indexed_status = 0';
$this->db()->exec($sSQL);
}
public function createSearchIndices()
{
info('Create Search indices');