safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

new PHP Nominatim\Shell class to wrap shell escaping

Browse Source
This commit is contained in:
marc tobias
2020-02-18 19:42:15 +01:00
parent 553d8a828c
commit fc40939775
5 changed files with 381 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
lib/Shell.php Normal file
View File

@@ -0,0 +1,80 @@
<?php
namespace Nominatim;
class Shell
{
public function __construct($sBaseCmd, ...$aParams)
{
if (!$sBaseCmd) {
throw new Exception('Command missing in new() call');
}
$this->baseCmd = $sBaseCmd;
$this->aParams = array();
$this->aEnv = null; // null = use the same environment as the current PHP process
$this->stdoutString = null;
foreach ($aParams as $sParam) {
$this->addParams($sParam);
}
}
public function addParams(...$aParams)
{
foreach ($aParams as $sParam) {
if (isset($sParam) && $sParam !== null && $sParam !== '') {
array_push($this->aParams, $sParam);
}
}
return $this;
}
public function addEnvPair($sKey, $sVal)
{
if (isset($sKey) && $sKey && isset($sVal)) {
if (!isset($this->aEnv)) $this->aEnv = $_ENV;
$this->aEnv = array_merge($this->aEnv, array($sKey => $sVal), $_ENV);
}
return $this;
}
public function escapedCmd()
{
$aEscaped = array_map(function ($sParam) {
return $this->escapeParam($sParam);
}, array_merge(array($this->baseCmd), $this->aParams));
return join(' ', $aEscaped);
}
public function run()
{
$sCmd = $this->escapedCmd();
// $aEnv does not need escaping, proc_open seems to handle it fine
$aFDs = array(
0 => array('pipe', 'r'),
1 => STDOUT,
2 => STDERR
);
$aPipes = null;
$hProc = @proc_open($sCmd, $aFDs, $aPipes, null, $this->aEnv);
if (!is_resource($hProc)) {
throw new \Exception('Unable to run command: ' . $sCmd);
}
fclose($aPipes[0]); // no stdin
$iStat = proc_close($hProc);
return $iStat;
}
private function escapeParam($sParam)
{
if (preg_match('/^-*\w+$/', $sParam)) return $sParam;
return escapeshellarg($sParam);
}
}

44
lib/cmd.php
View File

@@ -1,5 +1,6 @@
<?php
require_once(CONST_BasePath.'/lib/Shell.php');
function getCmdOpt($aArg, $aSpec, &$aResult, $bExitOnError = false, $bExitOnUnknown = false)
{
@@ -148,32 +149,33 @@ function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreError
// Convert database DSN to psql parameters
$aDSNInfo = \Nominatim\DB::parseDSN(CONST_Database_DSN);
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432;
$sCMD = 'psql'
.' -p '.escapeshellarg($aDSNInfo['port'])
.' -d '.escapeshellarg($aDSNInfo['database']);
$oCmd = new \Nominatim\Shell('psql');
$oCmd->addParams('--port', $aDSNInfo['port']);
$oCmd->addParams('--dbname', $aDSNInfo['database']);
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
$sCMD .= ' -h ' . escapeshellarg($aDSNInfo['hostspec']);
$oCmd->addParams('--host', $aDSNInfo['hostspec']);
}
if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
$sCMD .= ' -U ' . escapeshellarg($aDSNInfo['username']);
$oCmd->addParams('--username', $aDSNInfo['username']);
}
$aProcEnv = null;
if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV);
if (isset($aDSNInfo['password'])) {
$oCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']);
}
if (!$bVerbose) {
$sCMD .= ' -q';
$oCmd->addParams('--quiet');
}
if ($bfatal && !$bIgnoreErrors) {
$sCMD .= ' -v ON_ERROR_STOP=1';
$oCmd->addParams('-v', 'ON_ERROR_STOP=1');
}
$aDescriptors = array(
0 => array('pipe', 'r'),
1 => STDOUT,
2 => STDERR
);
$ahPipes = null;
$hProcess = @proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
$hProcess = @proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
if (!is_resource($hProcess)) {
fail('unable to start pgsql');
}
@@ -193,23 +195,3 @@ function runSQLScript($sScript, $bfatal = true, $bVerbose = false, $bIgnoreError
fail("pgsql returned with error code ($iReturn)");
}
}
function runWithEnv($sCmd, $aEnv)
{
$aFDs = array(
0 => array('pipe', 'r'),
1 => STDOUT,
2 => STDERR
);
$aPipes = null;
$hProc = @proc_open($sCmd, $aFDs, $aPipes, null, $aEnv);
if (!is_resource($hProc)) {
fail('unable to run command:' . $sCmd);
}
fclose($aPipes[0]); // no stdin
$iStat = proc_close($hProc);
return $iStat;
}

148
lib/setup/SetupClass.php
View File

@@ -3,6 +3,7 @@
namespace Nominatim\Setup;
require_once(CONST_BasePath.'/lib/setup/AddressLevelParser.php');
require_once(CONST_BasePath.'/lib/Shell.php');
class SetupFunctions
{
@@ -51,7 +52,7 @@ class SetupFunctions
}
// setting member variables based on command line options stored in $aCMDResult
$this->bQuiet = $aCMDResult['quiet'];
$this->bQuiet = isset($aCMDResult['quiet']) && $aCMDResult['quiet'];
$this->bVerbose = $aCMDResult['verbose'];
//setting default values which are not set by the update.php array
@@ -76,7 +77,7 @@ class SetupFunctions
$this->bEnableDiffUpdates = false;
}
$this->bDrop = $aCMDResult['drop'];
$this->bDrop = isset($aCMDResult['drop']) && $aCMDResult['drop'];
}
public function createDB()
@@ -88,19 +89,23 @@ class SetupFunctions
fail('database already exists ('.CONST_Database_DSN.')');
}
$sCreateDBCmd = 'createdb -E UTF-8'
.' -p '.escapeshellarg($this->aDSNInfo['port'])
.' '.escapeshellarg($this->aDSNInfo['database']);
$oCmd = (new \Nominatim\Shell('createdb'))
->addParams('-E', 'UTF-8')
->addParams('-p', $this->aDSNInfo['port']);
if (isset($this->aDSNInfo['username'])) {
$sCreateDBCmd .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
$oCmd->addParams('-U', $this->aDSNInfo['username']);
}
if (isset($this->aDSNInfo['password'])) {
$oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
}
if (isset($this->aDSNInfo['hostspec'])) {
$sCreateDBCmd .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']);
$oCmd->addParams('-h', $this->aDSNInfo['hostspec']);
}
$oCmd->addParams($this->aDSNInfo['database']);
$result = $this->runWithPgEnv($sCreateDBCmd);
if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd);
$result = $oCmd->run();
if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd());
}
public function connect()
@@ -174,39 +179,49 @@ class SetupFunctions
{
info('Import data');
$osm2pgsql = CONST_Osm2pgsql_Binary;
if (!file_exists($osm2pgsql)) {
if (!file_exists(CONST_Osm2pgsql_Binary)) {
echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n";
echo "Normally you should not need to set this manually.\n";
fail("osm2pgsql not found in '$osm2pgsql'");
fail("osm2pgsql not found in '".CONST_Osm2pgsql_Binary."'");
}
$osm2pgsql .= ' -S '.escapeshellarg(CONST_Import_Style);
$oCmd = new \Nominatim\Shell(CONST_Osm2pgsql_Binary);
$oCmd->addParams('--style', CONST_Import_Style);
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
$osm2pgsql .= ' --flat-nodes '.escapeshellarg(CONST_Osm2pgsql_Flatnode_File);
$oCmd->addParams('--flat-nodes', CONST_Osm2pgsql_Flatnode_File);
}
if (CONST_Tablespace_Osm2pgsql_Data) {
$oCmd->addParams('--tablespace-slim-data', CONST_Tablespace_Osm2pgsql_Data);
}
if (CONST_Tablespace_Osm2pgsql_Index) {
$oCmd->addParams('--tablespace-slim-index', CONST_Tablespace_Osm2pgsql_Index);
}
if (CONST_Tablespace_Place_Data) {
$oCmd->addParams('--tablespace-main-data', CONST_Tablespace_Place_Data);
}
if (CONST_Tablespace_Place_Index) {
$oCmd->addParams('--tablespace-main-index', CONST_Tablespace_Place_Index);
}
$oCmd->addParams('--latlong', '--slim', '--create');
$oCmd->addParams('--output', 'gazetteer');
$oCmd->addParams('--hstore');
$oCmd->addParams('--number-processes', 1);
$oCmd->addParams('--cache', $this->iCacheMemory);
$oCmd->addParams('--port', $this->aDSNInfo['port']);
if (CONST_Tablespace_Osm2pgsql_Data)
$osm2pgsql .= ' --tablespace-slim-data '.escapeshellarg(CONST_Tablespace_Osm2pgsql_Data);
if (CONST_Tablespace_Osm2pgsql_Index)
$osm2pgsql .= ' --tablespace-slim-index '.escapeshellarg(CONST_Tablespace_Osm2pgsql_Index);
if (CONST_Tablespace_Place_Data)
$osm2pgsql .= ' --tablespace-main-data '.escapeshellarg(CONST_Tablespace_Place_Data);
if (CONST_Tablespace_Place_Index)
$osm2pgsql .= ' --tablespace-main-index '.escapeshellarg(CONST_Tablespace_Place_Index);
$osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
$osm2pgsql .= ' -C '.escapeshellarg($this->iCacheMemory);
$osm2pgsql .= ' -P '.escapeshellarg($this->aDSNInfo['port']);
if (isset($this->aDSNInfo['username'])) {
$osm2pgsql .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
$oCmd->addParams('--username', $this->aDSNInfo['username']);
}
if (isset($this->aDSNInfo['password'])) {
$oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
}
if (isset($this->aDSNInfo['hostspec'])) {
$osm2pgsql .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']);
$oCmd->addParams('--host', $this->aDSNInfo['hostspec']);
}
$osm2pgsql .= ' -d '.escapeshellarg($this->aDSNInfo['database']).' '.escapeshellarg($sOSMFile);
$this->runWithPgEnv($osm2pgsql);
$oCmd->addParams('--database', $this->aDSNInfo['database']);
$oCmd->addParams($sOSMFile);
$oCmd->run();
if (!$this->sIgnoreErrors && !$this->oDB->getRow('select * from place limit 1')) {
fail('No Data');
@@ -529,39 +544,48 @@ class SetupFunctions
public function index($bIndexNoanalyse)
{
$sBaseCmd = CONST_BasePath.'/nominatim/nominatim.py'
.' -d '.escapeshellarg($this->aDSNInfo['database'])
.' -P '.escapeshellarg($this->aDSNInfo['port'])
.' -t '.escapeshellarg($this->iInstances);
$oBaseCmd = (new \Nominatim\Shell(CONST_BasePath.'/nominatim/nominatim.py'))
->addParams('--database', $this->aDSNInfo['database'])
->addParams('--port', $this->aDSNInfo['port'])
->addParams('--threads', $this->iInstances);
if (!$this->bQuiet) {
$sBaseCmd .= ' -v';
$oBaseCmd->addParams('-v');
}
if ($this->bVerbose) {
$sBaseCmd .= ' -v';
$oBaseCmd->addParams('-v');
}
if (isset($this->aDSNInfo['hostspec'])) {
$sBaseCmd .= ' -H '.escapeshellarg($this->aDSNInfo['hostspec']);
$oBaseCmd->addParams('--host', $this->aDSNInfo['hostspec']);
}
if (isset($this->aDSNInfo['username'])) {
$sBaseCmd .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
$oBaseCmd->addParams('--user', $this->aDSNInfo['username']);
}
if (isset($this->aDSNInfo['password'])) {
$oBaseCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
}
info('Index ranks 0 - 4');
$iStatus = $this->runWithPgEnv($sBaseCmd.' -R 4');
$oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4);
echo $oCmd->escapedCmd();
$iStatus = $oCmd->run();
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
info('Index ranks 5 - 25');
$iStatus = $this->runWithPgEnv($sBaseCmd.' -r 5 -R 25');
$oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25);
$iStatus = $oCmd->run();
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
info('Index ranks 26 - 30');
$iStatus = $this->runWithPgEnv($sBaseCmd.' -r 26');
$oCmd = (clone $oBaseCmd)->addParams('--minrank', 26);
$iStatus = $oCmd->run();
if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!');
}
@@ -753,21 +777,21 @@ class SetupFunctions
{
if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
$sCMD = 'psql'
.' -p '.escapeshellarg($this->aDSNInfo['port'])
.' -d '.escapeshellarg($this->aDSNInfo['database']);
$oCmd = (new \Nominatim\Shell('psql'))
->addParams('--port', $this->aDSNInfo['port'])
->addParams('--dbname', $this->aDSNInfo['database']);
if (!$this->bVerbose) {
$sCMD .= ' -q';
$oCmd->addParams('--quiet');
}
if (isset($this->aDSNInfo['hostspec'])) {
$sCMD .= ' -h '.escapeshellarg($this->aDSNInfo['hostspec']);
$oCmd->addParams('--host', $this->aDSNInfo['hostspec']);
}
if (isset($this->aDSNInfo['username'])) {
$sCMD .= ' -U '.escapeshellarg($this->aDSNInfo['username']);
$oCmd->addParams('--username', $this->aDSNInfo['username']);
}
$aProcEnv = null;
if (isset($this->aDSNInfo['password'])) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
$oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
}
$ahGzipPipes = null;
if (preg_match('/\\.gz$/', $sFilename)) {
@@ -776,12 +800,14 @@ class SetupFunctions
1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a')
);
$hGzipProcess = proc_open('zcat '.escapeshellarg($sFilename), $aDescriptors, $ahGzipPipes);
$oZcatCmd = new \Nominatim\Shell('zcat', $sFilename);
$hGzipProcess = proc_open($oZcatCmd->escapedCmd(), $aDescriptors, $ahGzipPipes);
if (!is_resource($hGzipProcess)) fail('unable to start zcat');
$aReadPipe = $ahGzipPipes[1];
fclose($ahGzipPipes[0]);
} else {
$sCMD .= ' -f '.escapeshellarg($sFilename);
$oCmd->addParams('--file', $sFilename);
$aReadPipe = array('pipe', 'r');
}
$aDescriptors = array(
@@ -790,7 +816,8 @@ class SetupFunctions
2 => array('file', '/dev/null', 'a')
);
$ahPipes = null;
$hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
$hProcess = proc_open($oCmd->escapedCmd(), $aDescriptors, $ahPipes, null, $oCmd->aEnv);
if (!is_resource($hProcess)) fail('unable to start pgsql');
// TODO: error checking
while (!feof($ahPipes[1])) {
@@ -831,21 +858,6 @@ class SetupFunctions
return $sSql;
}
private function runWithPgEnv($sCmd)
{
if ($this->bVerbose) {
echo "Execute: $sCmd\n";
}
$aProcEnv = null;
if (isset($this->aDSNInfo['password'])) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
}
return runWithEnv($sCmd, $aProcEnv);
}
/**
* Drop table with the given name if it exists.
*