From 14aca11dcd4c9778cac78f14c9c51c6bbe00bf96 Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Fri, 24 Aug 2018 16:15:39 +0200 Subject: [PATCH 01/12] moving functions from setup.php to a lib file in lib/setup_functions.php and change a passthru in setup.php by calling the function with this new lib --- lib/setup_functions.php | 965 ++++++++++++++++++++++++++++++++++++++++ utils/setup.php | 912 +------------------------------------ utils/update.php | 9 +- 3 files changed, 991 insertions(+), 895 deletions(-) create mode 100755 lib/setup_functions.php diff --git a/lib/setup_functions.php b/lib/setup_functions.php new file mode 100755 index 00000000..b6d927ab --- /dev/null +++ b/lib/setup_functions.php @@ -0,0 +1,965 @@ + $aDSNInfo['password']), $_ENV); + } + + $result = runWithEnv($sCreateDBCmd, $aProcEnv); + if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); + } + + if ($aCMDResult['setup-db'] || $aCMDResult['all']) { + info('Setup DB'); + $bDidSomething = true; + + $oDB =& getDB(); + + $fPostgresVersion = getPostgresVersion($oDB); + echo 'Postgres version found: '.$fPostgresVersion."\n"; + + if ($fPostgresVersion < 9.1) { + fail('Minimum supported version of Postgresql is 9.1.'); + } + + pgsqlRunScript($aCMDResult, 'CREATE EXTENSION IF NOT EXISTS hstore'); + pgsqlRunScript($aCMDResult, 'CREATE EXTENSION IF NOT EXISTS postgis'); + + // For extratags and namedetails the hstore_to_json converter is + // needed which is only available from Postgresql 9.3+. For older + // versions add a dummy function that returns nothing. + $iNumFunc = chksql($oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); + + if ($iNumFunc == 0) { + pgsqlRunScript($aCMDResult, "create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); + warn('Postgresql is too old. extratags and namedetails API not available.'); + } + + $fPostgisVersion = getPostgisVersion($oDB); + echo 'Postgis version found: '.$fPostgisVersion."\n"; + + if ($fPostgisVersion < 2.1) { + // Functions were renamed in 2.1 and throw an annoying deprecation warning + pgsqlRunScript($aCMDResult, 'ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint'); + pgsqlRunScript($aCMDResult, 'ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint'); + } + if ($fPostgisVersion < 2.2) { + pgsqlRunScript($aCMDResult, 'ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); + } + + $i = chksql($oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'")); + if ($i == 0) { + echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; + echo "\n createuser ".CONST_Database_Web_User."\n\n"; + exit(1); + } + + if (!checkModulePresence()) { + fail('error loading nominatim.so module'); + } + + if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { + echo 'Error: you need to download the country_osm_grid first:'; + echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; + exit(1); + } + pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/country_name.sql'); + pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/country_naturalearthdata.sql'); + pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/country_osm_grid.sql.gz'); + pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/gb_postcode_table.sql'); + + + if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { + pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/gb_postcode_data.sql.gz'); + } else { + warn('external UK postcode table not found.'); + } + + if (CONST_Use_Extra_US_Postcodes) { + pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/us_postcode.sql'); + } + + if ($aCMDResult['no-partitions']) { + pgsqlRunScript($aCMDResult, 'update country_name set partition = 0'); + } + + // the following will be needed by create_functions later but + // is only defined in the subsequently called T + // Create dummies here that will be overwritten by the proper + // versions in create-tables. + pgsqlRunScript($aCMDResult, 'CREATE TABLE IF NOT EXISTS place_boundingbox ()'); + pgsqlRunScript($aCMDResult, 'CREATE TYPE wikipedia_article_match AS ()', false); + } + return $aDSNInfo; +} + +function importData($aCMDResult, $iCacheMemory, $aDSNInfo) +{ + info('Import data'); + + $osm2pgsql = CONST_Osm2pgsql_Binary; + if (!file_exists($osm2pgsql)) { + echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; + echo "Normally you should not need to set this manually.\n"; + fail("osm2pgsql not found in '$osm2pgsql'"); + } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { + $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; + } + if (CONST_Tablespace_Osm2pgsql_Data) + $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data; + if (CONST_Tablespace_Osm2pgsql_Index) + $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index; + if (CONST_Tablespace_Place_Data) + $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data; + if (CONST_Tablespace_Place_Index) + $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; + $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; + $osm2pgsql .= ' -C '.$iCacheMemory; + $osm2pgsql .= ' -P '.$aDSNInfo['port']; + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $osm2pgsql .= ' -U ' . $aDSNInfo['username']; + } + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $osm2pgsql .= ' -H ' . $aDSNInfo['hostspec']; + } + + $aProcEnv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } + + $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; + runWithEnv($osm2pgsql, $aProcEnv); + + $oDB =& getDB(); + if (!$aCMDResult['ignore-errors'] && !chksql($oDB->getRow('select * from place limit 1'))) { + fail('No Data'); + } +} + +function createFunctions($aCMDResult) +{ + info('Create Functions'); + + if (!checkModulePresence()) { + fail('error loading nominatim.so module'); + } + + createSqlFunctions($aCMDResult); +} + +function createTables($aCMDResult) +{ + info('Create Tables'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = replaceTablespace( + '{ts:address-data}', + CONST_Tablespace_Address_Data, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:search-data}', + CONST_Tablespace_Search_Data, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + pgsqlRunScript($aCMDResult, $sTemplate, false); + + // re-run the functions + info('Recreate Functions'); + createSqlFunctions($aCMDResult); +} + +function createPartitionTables($aCMDResult) +{ + info('Create Partition Tables'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); + $sTemplate = replaceTablespace( + '{ts:address-data}', + CONST_Tablespace_Address_Data, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:search-data}', + CONST_Tablespace_Search_Data, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + + pgsqlRunPartitionScript($aCMDResult, $sTemplate); +} + +function createPartitionFunctions($aCMDResult) +{ + info('Create Partition Functions'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); + + pgsqlRunPartitionScript($aCMDResult, $sTemplate); +} + +function importWikipediaArticles() +{ + $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; + $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; + if (file_exists($sWikiArticlesFile)) { + info('Importing wikipedia articles'); + pgsqlRunDropAndRestore($sWikiArticlesFile); + } else { + warn('wikipedia article dump file not found - places will have default importance'); + } + if (file_exists($sWikiRedirectsFile)) { + info('Importing wikipedia redirects'); + pgsqlRunDropAndRestore($sWikiRedirectsFile); + } else { + warn('wikipedia redirect dump file not found - some place importance values may be missing'); + } +} + +function loadData($aCMDResult, $iInstances) +{ + info('Drop old Data'); + + $oDB =& getDB(); + if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection)); + echo '.'; + if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection)); + echo '.'; + if (!pg_query($oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($oDB->connection)); + echo '.'; + if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection)); + echo '.'; + if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection)); + echo '.'; + if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection)); + echo '.'; + if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection)); + echo '.'; + if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection)); + echo '.'; + if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection)); + echo '.'; + if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection)); + echo '.'; + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($oDB->getCol($sSQL)); + if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; + foreach ($aPartitions as $sPartition) { + if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection)); + echo '.'; + } + + // used by getorcreate_word_id to ignore frequent partial words + $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; + $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; + if (!pg_query($oDB->connection, $sSQL)) { + fail(pg_last_error($oDB->connection)); + } + echo ".\n"; + + // pre-create the word list + if (!$aCMDResult['disable-token-precalc']) { + info('Loading word list'); + pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/words.sql'); + } + + info('Load Data'); + $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; + + $aDBInstances = array(); + $iLoadThreads = max(1, $iInstances - 1); + for ($i = 0; $i < $iLoadThreads; $i++) { + $aDBInstances[$i] =& getDB(true); + $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; + $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; + $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; + $sSQL .= ' and ST_IsValid(geometry)'; + if ($aCMDResult['verbose']) echo "$sSQL\n"; + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { + fail(pg_last_error($aDBInstances[$i]->connection)); + } + } + // last thread for interpolation lines + $aDBInstances[$iLoadThreads] =& getDB(true); + $sSQL = 'insert into location_property_osmline'; + $sSQL .= ' (osm_id, address, linegeo)'; + $sSQL .= ' SELECT osm_id, address, geometry from place where '; + $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; + if ($aCMDResult['verbose']) echo "$sSQL\n"; + if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { + fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); + } + + $bFailed = false; + for ($i = 0; $i <= $iLoadThreads; $i++) { + while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { + $resultStatus = pg_result_status($hPGresult); + // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, + // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, + // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR + echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; + if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { + $resultError = pg_result_error($hPGresult); + echo '-- error text ' . $i . ': ' . $resultError . "\n"; + $bFailed = true; + } + } + } + if ($bFailed) { + fail('SQL errors loading placex and/or location_property_osmline tables'); + } + echo "\n"; + info('Reanalysing database'); + pgsqlRunScript($aCMDResult, 'ANALYSE'); + + $sDatabaseDate = getDatabaseDate($oDB); + pg_query($oDB->connection, 'TRUNCATE import_status'); + if ($sDatabaseDate === false) { + warn('could not determine database date.'); + } else { + $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; + pg_query($oDB->connection, $sSQL); + echo "Latest data imported from $sDatabaseDate.\n"; + } +} + +function importTigerData($aCMDResult, $iInstances) +{ + info('Import Tiger data'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + pgsqlRunScript($aCMDResult, $sTemplate, false); + + $aDBInstances = array(); + for ($i = 0; $i < $iInstances; $i++) { + $aDBInstances[$i] =& getDB(true); + } + + foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) { + echo $sFile.': '; + $hFile = fopen($sFile, 'r'); + $sSQL = fgets($hFile, 100000); + $iLines = 0; + + while (true) { + for ($i = 0; $i < $iInstances; $i++) { + if (!pg_connection_busy($aDBInstances[$i]->connection)) { + while (pg_get_result($aDBInstances[$i]->connection)); + $sSQL = fgets($hFile, 100000); + if (!$sSQL) break 2; + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection)); + $iLines++; + if ($iLines == 1000) { + echo '.'; + $iLines = 0; + } + } + } + usleep(10); + } + + fclose($hFile); + + $bAnyBusy = true; + while ($bAnyBusy) { + $bAnyBusy = false; + for ($i = 0; $i < $iInstances; $i++) { + if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; + } + usleep(10); + } + echo "\n"; + } + + info('Creating indexes on Tiger data'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + pgsqlRunScript($aCMDResult, $sTemplate, false); +} + +function calculatePostcodes($aCMDResult) +{ + info('Calculate Postcodes'); + $oDB =& getDB(); + if (!pg_query($oDB->connection, 'TRUNCATE location_postcode')) { + fail(pg_last_error($oDB->connection)); + } + + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; + $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; + $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; + $sSQL .= ' FROM placex'; + $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; + $sSQL .= ' AND geometry IS NOT null'; + $sSQL .= ' GROUP BY country_code, pc'; + + if (!pg_query($oDB->connection, $sSQL)) { + fail(pg_last_error($oDB->connection)); + } + + if (CONST_Use_Extra_US_Postcodes) { + // only add postcodes that are not yet available in OSM + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; + $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; + $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; + $sSQL .= ' (SELECT postcode FROM location_postcode'; + $sSQL .= " WHERE country_code = 'us')"; + if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); + } + + // add missing postcodes for GB (if available) + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; + $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; + $sSQL .= ' (SELECT postcode FROM location_postcode'; + $sSQL .= " WHERE country_code = 'gb')"; + if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); + + if (!$aCMDResult['all']) { + $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; + $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; + if (!pg_query($oDB->connection, $sSQL)) { + fail(pg_last_error($oDB->connection)); + } + } + $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; + $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; + + if (!pg_query($oDB->connection, $sSQL)) { + fail(pg_last_error($oDB->connection)); + } +} + +function index($aCMDResult, $aDSNInfo, $iInstances) +{ + $sOutputFile = ''; + $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile; + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sBaseCmd .= ' -H ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sBaseCmd .= ' -U ' . $aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } + + info('Index ranks 0 - 4'); + $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + if (!$aCMDResult['index-noanalyse']) pgsqlRunScript($aCMDResult, 'ANALYSE'); + info('Index ranks 5 - 25'); + $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + if (!$aCMDResult['index-noanalyse']) pgsqlRunScript($aCMDResult, 'ANALYSE'); + info('Index ranks 26 - 30'); + $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + + info('Index postcodes'); + $oDB =& getDB(); + $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; + if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); +} + +function createSearchIndices($aCMDResult) +{ + info('Create Search indices'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + + pgsqlRunScript($aCMDResult, $sTemplate); +} + +function createCountryNames($aCMDResult) +{ + info('Create search index for default country names'); + + pgsqlRunScript($aCMDResult, "select getorcreate_country(make_standard_name('uk'), 'gb')"); + pgsqlRunScript($aCMDResult, "select getorcreate_country(make_standard_name('united states'), 'us')"); + pgsqlRunScript($aCMDResult, 'select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); + pgsqlRunScript($aCMDResult, "select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); + + $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v), country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; + if (CONST_Languages) { + $sSQL .= 'in '; + $sDelim = '('; + foreach (explode(',', CONST_Languages) as $sLang) { + $sSQL .= $sDelim."'name:$sLang'"; + $sDelim = ','; + } + $sSQL .= ')'; + } else { + // all include all simple name tags + $sSQL .= "like 'name:%'"; + } + $sSQL .= ') v'; + pgsqlRunScript($aCMDResult, $sSQL); +} + +function drop($aCMDResult) +{ + info('Drop tables only required for updates'); + // The implementation is potentially a bit dangerous because it uses + // a positive selection of tables to keep, and deletes everything else. + // Including any tables that the unsuspecting user might have manually + // created. USE AT YOUR OWN PERIL. + + + // tables we want to keep. everything else goes. + $aKeepTables = array( + '*columns', + 'import_polygon_*', + 'import_status', + 'place_addressline', + 'location_postcode', + 'location_property*', + 'placex', + 'search_name', + 'seq_*', + 'word', + 'query_log', + 'new_query_log', + 'spatial_ref_sys', + 'country_name', + 'place_classtype_*' + ); + + $oDB =& getDB(); + $aDropTables = array(); + $aHaveTables = chksql($oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); + + foreach ($aHaveTables as $sTable) { + $bFound = false; + foreach ($aKeepTables as $sKeep) { + if (fnmatch($sKeep, $sTable)) { + $bFound = true; + break; + } + } + if (!$bFound) array_push($aDropTables, $sTable); + } + + foreach ($aDropTables as $sDrop) { + if ($aCMDResult['verbose']) echo "dropping table $sDrop\n"; + @pg_query($oDB->connection, "DROP TABLE $sDrop CASCADE"); + // ignore warnings/errors as they might be caused by a table having + // been deleted already by CASCADE + } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { + if ($aCMDResult['verbose']) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; + unlink(CONST_Osm2pgsql_Flatnode_File); + } +} + +function didSomething($bDidSomething) +{ + if (!$bDidSomething) { + showUsage($aCMDOptions, true); + } else { + echo "Summary of warnings:\n\n"; + repeatWarnings(); + echo "\n"; + info('Setup finished.'); + } +} + +// ********************************* + +function pgsqlRunScriptFile($aCMDResult, $sFilename) +{ + if (!file_exists($sFilename)) fail('unable to find '.$sFilename); + // Convert database DSN to psql parameters + $aDSNInfo = DB::parseDSN(CONST_Database_DSN); + if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; + $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database']; + if (!$aCMDResult['verbose']) { + $sCMD .= ' -q'; + } + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCMD .= ' -U ' . $aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } + $ahGzipPipes = null; + if (preg_match('/\\.gz$/', $sFilename)) { + $aDescriptors = array( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a') + ); + $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); + if (!is_resource($hGzipProcess)) fail('unable to start zcat'); + $aReadPipe = $ahGzipPipes[1]; + fclose($ahGzipPipes[0]); + } else { + $sCMD .= ' -f '.$sFilename; + $aReadPipe = array('pipe', 'r'); + } + $aDescriptors = array( + 0 => $aReadPipe, + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a') + ); + $ahPipes = null; + $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); + if (!is_resource($hProcess)) fail('unable to start pgsql'); + // TODO: error checking + while (!feof($ahPipes[1])) { + echo fread($ahPipes[1], 4096); + } + fclose($ahPipes[1]); + $iReturn = proc_close($hProcess); + if ($iReturn > 0) { + fail("pgsql returned with error code ($iReturn)"); + } + if ($ahGzipPipes) { + fclose($ahGzipPipes[1]); + proc_close($hGzipProcess); + } +} + +function pgsqlRunScript($aCMDResult, $sScript, $bfatal = true) +{ + runSQLScript( + $sScript, + $bfatal, + $aCMDResult['verbose'], + $aCMDResult['ignore-errors'] + ); +} + +function pgsqlRunPartitionScript($aCMDResult, $sTemplate) +{ + global $aCMDResult; + $oDB =& getDB(); + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($oDB->getCol($sSQL)); + if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; + + preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); + foreach ($aMatches as $aMatch) { + $sResult = ''; + foreach ($aPartitions as $sPartitionName) { + $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); + } + $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); + } + + pgsqlRunScript($aCMDResult, $sTemplate); +} + +function pgsqlRunRestoreData($sDumpFile) +{ + // Convert database DSN to psql parameters + $aDSNInfo = DB::parseDSN(CONST_Database_DSN); + if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; + $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile; + + $aDescriptors = array( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a') + ); + $ahPipes = null; + $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes); + if (!is_resource($hProcess)) fail('unable to start pg_restore'); + + fclose($ahPipes[0]); + + // TODO: error checking + while (!feof($ahPipes[1])) { + echo fread($ahPipes[1], 4096); + } + fclose($ahPipes[1]); + + $iReturn = proc_close($hProcess); +} + +function pgsqlRunDropAndRestore($sDumpFile) +{ + // Convert database DSN to psql parameters + $aDSNInfo = DB::parseDSN(CONST_Database_DSN); + if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; + $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile; + if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $aDSNInfo['hostspec']; + } + if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { + $sCMD .= ' -U ' . $aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); + } + + $iReturn = runWithEnv($sCMD, $aProcEnv); +} + +function passthpassthruCheckReturn($sCmd) +{ + $iResult = -1; + passthru($sCmd, $iResult); +} + +function replaceTablespace($sTemplate, $sTablespace, $sSql) +{ + if ($sTablespace) { + $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); + } else { + $sSql = str_replace($sTemplate, '', $sSql); + } + + return $sSql; +} + +function createSqlFunctions($aCMDResult) +{ + $sModulePath = CONST_Database_Module_Path; + $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); + $sTemplate = str_replace('{modulepath}', $sModulePath, $sTemplate); + if ($aCMDResult['enable-diff-updates']) { + $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); + } + if ($aCMDResult['enable-debug-statements']) { + $sTemplate = str_replace('--DEBUG:', '', $sTemplate); + } + if (CONST_Limit_Reindexing) { + $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); + } + if (!CONST_Use_US_Tiger_Data) { + $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); + } + if (!CONST_Use_Aux_Location_data) { + $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); + } + pgsqlRunScript($aCMDResult, $sTemplate); +} + +function checkModulePresence() +{ + // Try accessing the C module, so we know early if something is wrong + // and can simply error out. + $sModulePath = CONST_Database_Module_Path; + $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '"; + $sSQL .= $sModulePath."/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; + $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);'; + + $oDB =& getDB(); + $oResult = $oDB->query($sSQL); + + $bResult = true; + + if (PEAR::isError($oResult)) { + echo "\nERROR: Failed to load nominatim module. Reason:\n"; + echo $oResult->userinfo."\n\n"; + $bResult = false; + } + + return $bResult; +} + +function createSetupArgvArray() +{ + $aCMDOptions + = array( + 'Create and setup nominatim search system', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), + array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), + + array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), + + array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), + array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), + array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), + array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), + array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), + array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), + array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), + array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), + array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), + array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), + array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), + array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), + array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), + array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), + array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), + array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), + array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), + array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), + array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), + array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), + ); + return $aCMDOptions; +} diff --git a/utils/setup.php b/utils/setup.php index b9c0166c..449dd4da 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -3,599 +3,72 @@ require_once(dirname(dirname(__FILE__)).'/settings/settings.php'); require_once(CONST_BasePath.'/lib/init-cmd.php'); +include_once(CONST_BasePath.'/lib/setup_functions.php'); ini_set('memory_limit', '800M'); # (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) -$aCMDOptions -= array( - 'Create and setup nominatim search system', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), +$aCMDOptions = createSetupArgvArray(); - array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), - array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), - - array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), - - array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), - array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), - array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), - array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), - array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), - array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), - array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), - array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), - array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), - array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), - array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), - array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), - array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), - array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), - array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), - array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), - array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), - array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), - array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), - array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), - ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); $bDidSomething = false; // Check if osm-file is set and points to a valid file if --all or --import-data is given -if ($aCMDResult['import-data'] || $aCMDResult['all']) { - if (!isset($aCMDResult['osm-file'])) { - fail('missing --osm-file for data import'); - } +checkInFile($aCMDResult); - if (!file_exists($aCMDResult['osm-file'])) { - fail('the path supplied to --osm-file does not exist'); - } +// get info how many processors and huch much cache mem we can use +$prepSreturn = prepSystem($aCMDResult); +$iCacheMemory = $prepSreturn[0]; +$iInstances = $prepSreturn[1]; - if (!is_readable($aCMDResult['osm-file'])) { - fail('osm-file "'.$aCMDResult['osm-file'].'" not readable'); - } -} - -// by default, use all but one processor, but never more than 15. -$iInstances = isset($aCMDResult['threads']) - ? $aCMDResult['threads'] - : (min(16, getProcessorCount()) - 1); - -if ($iInstances < 1) { - $iInstances = 1; - warn("resetting threads to $iInstances"); -} - -// Assume we can steal all the cache memory in the box (unless told otherwise) -if (isset($aCMDResult['osm2pgsql-cache'])) { - $iCacheMemory = $aCMDResult['osm2pgsql-cache']; -} else { - $iCacheMemory = getCacheMemoryMB(); -} - -$sModulePath = CONST_Database_Module_Path; -info('module path: ' . $sModulePath); - -$aDSNInfo = DB::parseDSN(CONST_Database_DSN); -if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - -if ($aCMDResult['create-db'] || $aCMDResult['all']) { - info('Create DB'); - $bDidSomething = true; - $oDB = DB::connect(CONST_Database_DSN, false); - if (!PEAR::isError($oDB)) { - fail('database already exists ('.CONST_Database_DSN.')'); - } - - $sCreateDBCmd = 'createdb -E UTF-8 -p '.$aDSNInfo['port'].' '.$aDSNInfo['database']; - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $sCreateDBCmd .= ' -U ' . $aDSNInfo['username']; - } - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $sCreateDBCmd .= ' -h ' . $aDSNInfo['hostspec']; - } - - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - $result = runWithEnv($sCreateDBCmd, $aProcEnv); - if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); -} - -if ($aCMDResult['setup-db'] || $aCMDResult['all']) { - info('Setup DB'); - $bDidSomething = true; - - $oDB =& getDB(); - - $fPostgresVersion = getPostgresVersion($oDB); - echo 'Postgres version found: '.$fPostgresVersion."\n"; - - if ($fPostgresVersion < 9.1) { - fail('Minimum supported version of Postgresql is 9.1.'); - } - - pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore'); - pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); - - // For extratags and namedetails the hstore_to_json converter is - // needed which is only available from Postgresql 9.3+. For older - // versions add a dummy function that returns nothing. - $iNumFunc = chksql($oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); - - if ($iNumFunc == 0) { - pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); - warn('Postgresql is too old. extratags and namedetails API not available.'); - } - - $fPostgisVersion = getPostgisVersion($oDB); - echo 'Postgis version found: '.$fPostgisVersion."\n"; - - if ($fPostgisVersion < 2.1) { - // Functions were renamed in 2.1 and throw an annoying deprecation warning - pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint'); - pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint'); - } - if ($fPostgisVersion < 2.2) { - pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); - } - - $i = chksql($oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'")); - if ($i == 0) { - echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; - echo "\n createuser ".CONST_Database_Web_User."\n\n"; - exit(1); - } - - if (!checkModulePresence()) { - fail('error loading nominatim.so module'); - } - - if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { - echo 'Error: you need to download the country_osm_grid first:'; - echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; - exit(1); - } - - pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); - pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); - pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz'); - pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql'); - if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { - pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz'); - } else { - warn('external UK postcode table not found.'); - } - if (CONST_Use_Extra_US_Postcodes) { - pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); - } - - if ($aCMDResult['no-partitions']) { - pgsqlRunScript('update country_name set partition = 0'); - } - - // the following will be needed by create_functions later but - // is only defined in the subsequently called create_tables. - // Create dummies here that will be overwritten by the proper - // versions in create-tables. - pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); - pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); -} +// prepares DB for import or update, returns Data Source Name +$aDSNInfo = prepDB($aCMDResult); if ($aCMDResult['import-data'] || $aCMDResult['all']) { - info('Import data'); $bDidSomething = true; - - $osm2pgsql = CONST_Osm2pgsql_Binary; - if (!file_exists($osm2pgsql)) { - echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; - echo "Normally you should not need to set this manually.\n"; - fail("osm2pgsql not found in '$osm2pgsql'"); - } - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; - } - if (CONST_Tablespace_Osm2pgsql_Data) - $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data; - if (CONST_Tablespace_Osm2pgsql_Index) - $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index; - if (CONST_Tablespace_Place_Data) - $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data; - if (CONST_Tablespace_Place_Index) - $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; - $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; - $osm2pgsql .= ' -C '.$iCacheMemory; - $osm2pgsql .= ' -P '.$aDSNInfo['port']; - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $osm2pgsql .= ' -U ' . $aDSNInfo['username']; - } - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $osm2pgsql .= ' -H ' . $aDSNInfo['hostspec']; - } - - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; - runWithEnv($osm2pgsql, $aProcEnv); - - $oDB =& getDB(); - if (!$aCMDResult['ignore-errors'] && !chksql($oDB->getRow('select * from place limit 1'))) { - fail('No Data'); - } + importData($aCMDResult, $iCacheMemory, $aDSNInfo); } if ($aCMDResult['create-functions'] || $aCMDResult['all']) { - info('Create Functions'); $bDidSomething = true; - - if (!checkModulePresence()) { - fail('error loading nominatim.so module'); - } - - create_sql_functions($aCMDResult); + createFunctions($aCMDResult); } if ($aCMDResult['create-tables'] || $aCMDResult['all']) { - info('Create Tables'); $bDidSomething = true; - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replace_tablespace( - '{ts:address-data}', - CONST_Tablespace_Address_Data, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:search-data}', - CONST_Tablespace_Search_Data, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - pgsqlRunScript($sTemplate, false); - - // re-run the functions - info('Recreate Functions'); - create_sql_functions($aCMDResult); + createTables($aCMDResult); } if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { - info('Create Partition Tables'); $bDidSomething = true; - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); - $sTemplate = replace_tablespace( - '{ts:address-data}', - CONST_Tablespace_Address_Data, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:search-data}', - CONST_Tablespace_Search_Data, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - - pgsqlRunPartitionScript($sTemplate); + createPartitionTables($aCMDResult); } - if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { - info('Create Partition Functions'); $bDidSomething = true; - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); - - pgsqlRunPartitionScript($sTemplate); + createPartitionFunctions($aCMDResult); } if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { $bDidSomething = true; - $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; - $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; - if (file_exists($sWikiArticlesFile)) { - info('Importing wikipedia articles'); - pgsqlRunDropAndRestore($sWikiArticlesFile); - } else { - warn('wikipedia article dump file not found - places will have default importance'); - } - if (file_exists($sWikiRedirectsFile)) { - info('Importing wikipedia redirects'); - pgsqlRunDropAndRestore($sWikiRedirectsFile); - } else { - warn('wikipedia redirect dump file not found - some place importance values may be missing'); - } + importWikipediaArticles(); } if ($aCMDResult['load-data'] || $aCMDResult['all']) { - info('Drop old Data'); $bDidSomething = true; - - $oDB =& getDB(); - if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection)); - echo '.'; - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = chksql($oDB->getCol($sSQL)); - if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; - foreach ($aPartitions as $sPartition) { - if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection)); - echo '.'; - } - - // used by getorcreate_word_id to ignore frequent partial words - $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; - $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } - echo ".\n"; - - // pre-create the word list - if (!$aCMDResult['disable-token-precalc']) { - info('Loading word list'); - pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql'); - } - - info('Load Data'); - $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; - - $aDBInstances = array(); - $iLoadThreads = max(1, $iInstances - 1); - for ($i = 0; $i < $iLoadThreads; $i++) { - $aDBInstances[$i] =& getDB(true); - $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; - $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; - $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; - $sSQL .= ' and ST_IsValid(geometry)'; - if ($aCMDResult['verbose']) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { - fail(pg_last_error($aDBInstances[$i]->connection)); - } - } - // last thread for interpolation lines - $aDBInstances[$iLoadThreads] =& getDB(true); - $sSQL = 'insert into location_property_osmline'; - $sSQL .= ' (osm_id, address, linegeo)'; - $sSQL .= ' SELECT osm_id, address, geometry from place where '; - $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; - if ($aCMDResult['verbose']) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { - fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); - } - - $bFailed = false; - for ($i = 0; $i <= $iLoadThreads; $i++) { - while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { - $resultStatus = pg_result_status($hPGresult); - // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, - // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, - // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR - if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { - $resultError = pg_result_error($hPGresult); - echo '-- error text ' . $i . ': ' . $resultError . "\n"; - $bFailed = true; - } - } - } - if ($bFailed) { - fail('SQL errors loading placex and/or location_property_osmline tables'); - } - echo "\n"; - info('Reanalysing database'); - pgsqlRunScript('ANALYSE'); - - $sDatabaseDate = getDatabaseDate($oDB); - pg_query($oDB->connection, 'TRUNCATE import_status'); - if ($sDatabaseDate === false) { - warn('could not determine database date.'); - } else { - $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; - pg_query($oDB->connection, $sSQL); - echo "Latest data imported from $sDatabaseDate.\n"; - } + loadData($aCMDResult, $iInstances); } if ($aCMDResult['import-tiger-data']) { - info('Import Tiger data'); $bDidSomething = true; - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replace_tablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - pgsqlRunScript($sTemplate, false); - - $aDBInstances = array(); - for ($i = 0; $i < $iInstances; $i++) { - $aDBInstances[$i] =& getDB(true); - } - - foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) { - echo $sFile.': '; - $hFile = fopen($sFile, 'r'); - $sSQL = fgets($hFile, 100000); - $iLines = 0; - - while (true) { - for ($i = 0; $i < $iInstances; $i++) { - if (!pg_connection_busy($aDBInstances[$i]->connection)) { - while (pg_get_result($aDBInstances[$i]->connection)); - $sSQL = fgets($hFile, 100000); - if (!$sSQL) break 2; - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection)); - $iLines++; - if ($iLines == 1000) { - echo '.'; - $iLines = 0; - } - } - } - usleep(10); - } - - fclose($hFile); - - $bAnyBusy = true; - while ($bAnyBusy) { - $bAnyBusy = false; - for ($i = 0; $i < $iInstances; $i++) { - if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; - } - usleep(10); - } - echo "\n"; - } - - info('Creating indexes on Tiger data'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replace_tablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - pgsqlRunScript($sTemplate, false); + importTigerData($aCMDResult, $iInstances); } if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) { - info('Calculate Postcodes'); $bDidSomething = true; - $oDB =& getDB(); - if (!pg_query($oDB->connection, 'TRUNCATE location_postcode')) { - fail(pg_last_error($oDB->connection)); - } - - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; - $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; - $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; - $sSQL .= ' FROM placex'; - $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; - $sSQL .= ' AND geometry IS NOT null'; - $sSQL .= ' GROUP BY country_code, pc'; - - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } - - if (CONST_Use_Extra_US_Postcodes) { - // only add postcodes that are not yet available in OSM - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; - $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; - $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'us')"; - if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); - } - - // add missing postcodes for GB (if available) - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; - $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'gb')"; - if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); - - if (!$aCMDResult['all']) { - $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; - $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } - } - $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; - $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; - - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } + calculatePostcodes($aCMDResult); } if ($aCMDResult['osmosis-init']) { @@ -605,361 +78,22 @@ if ($aCMDResult['osmosis-init']) { if ($aCMDResult['index'] || $aCMDResult['all']) { $bDidSomething = true; - $sOutputFile = ''; - $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile; - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $sBaseCmd .= ' -H ' . $aDSNInfo['hostspec']; - } - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $sBaseCmd .= ' -U ' . $aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - info('Index ranks 0 - 4'); - $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE'); - info('Index ranks 5 - 25'); - $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$aCMDResult['index-noanalyse']) pgsqlRunScript('ANALYSE'); - info('Index ranks 26 - 30'); - $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - - info('Index postcodes'); - $oDB =& getDB(); - $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; - if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); + index($aCMDResult, $aDSNInfo, $iInstances); } if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { - info('Create Search indices'); $bDidSomething = true; - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replace_tablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = replace_tablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - - pgsqlRunScript($sTemplate); + createSearchIndices($aCMDResult); } if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { - info('Create search index for default country names'); $bDidSomething = true; - - pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); - pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); - pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); - pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); - - $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v), country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; - if (CONST_Languages) { - $sSQL .= 'in '; - $sDelim = '('; - foreach (explode(',', CONST_Languages) as $sLang) { - $sSQL .= $sDelim."'name:$sLang'"; - $sDelim = ','; - } - $sSQL .= ')'; - } else { - // all include all simple name tags - $sSQL .= "like 'name:%'"; - } - $sSQL .= ') v'; - pgsqlRunScript($sSQL); + createCountryNames($aCMDResult); } if ($aCMDResult['drop']) { - info('Drop tables only required for updates'); - // The implementation is potentially a bit dangerous because it uses - // a positive selection of tables to keep, and deletes everything else. - // Including any tables that the unsuspecting user might have manually - // created. USE AT YOUR OWN PERIL. $bDidSomething = true; - - // tables we want to keep. everything else goes. - $aKeepTables = array( - '*columns', - 'import_polygon_*', - 'import_status', - 'place_addressline', - 'location_postcode', - 'location_property*', - 'placex', - 'search_name', - 'seq_*', - 'word', - 'query_log', - 'new_query_log', - 'spatial_ref_sys', - 'country_name', - 'place_classtype_*' - ); - - $oDB =& getDB(); - $aDropTables = array(); - $aHaveTables = chksql($oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); - - foreach ($aHaveTables as $sTable) { - $bFound = false; - foreach ($aKeepTables as $sKeep) { - if (fnmatch($sKeep, $sTable)) { - $bFound = true; - break; - } - } - if (!$bFound) array_push($aDropTables, $sTable); - } - - foreach ($aDropTables as $sDrop) { - if ($aCMDResult['verbose']) echo "dropping table $sDrop\n"; - @pg_query($oDB->connection, "DROP TABLE $sDrop CASCADE"); - // ignore warnings/errors as they might be caused by a table having - // been deleted already by CASCADE - } - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - if ($aCMDResult['verbose']) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; - unlink(CONST_Osm2pgsql_Flatnode_File); - } + drop($aCMDResult); } -if (!$bDidSomething) { - showUsage($aCMDOptions, true); -} else { - echo "Summary of warnings:\n\n"; - repeatWarnings(); - echo "\n"; - info('Setup finished.'); -} - - -function pgsqlRunScriptFile($sFilename) -{ - global $aCMDResult; - if (!file_exists($sFilename)) fail('unable to find '.$sFilename); - - // Convert database DSN to psql parameters - $aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database']; - if (!$aCMDResult['verbose']) { - $sCMD .= ' -q'; - } - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $aDSNInfo['hostspec']; - } - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $sCMD .= ' -U ' . $aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - $ahGzipPipes = null; - if (preg_match('/\\.gz$/', $sFilename)) { - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); - if (!is_resource($hGzipProcess)) fail('unable to start zcat'); - $aReadPipe = $ahGzipPipes[1]; - fclose($ahGzipPipes[0]); - } else { - $sCMD .= ' -f '.$sFilename; - $aReadPipe = array('pipe', 'r'); - } - - $aDescriptors = array( - 0 => $aReadPipe, - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); - if (!is_resource($hProcess)) fail('unable to start pgsql'); - - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); - } - fclose($ahPipes[1]); - - $iReturn = proc_close($hProcess); - if ($iReturn > 0) { - fail("pgsql returned with error code ($iReturn)"); - } - if ($ahGzipPipes) { - fclose($ahGzipPipes[1]); - proc_close($hGzipProcess); - } -} - -function pgsqlRunScript($sScript, $bfatal = true) -{ - global $aCMDResult; - runSQLScript( - $sScript, - $bfatal, - $aCMDResult['verbose'], - $aCMDResult['ignore-errors'] - ); -} - -function pgsqlRunPartitionScript($sTemplate) -{ - global $aCMDResult; - $oDB =& getDB(); - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = chksql($oDB->getCol($sSQL)); - if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; - - preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); - foreach ($aMatches as $aMatch) { - $sResult = ''; - foreach ($aPartitions as $sPartitionName) { - $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); - } - $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); - } - - pgsqlRunScript($sTemplate); -} - -function pgsqlRunRestoreData($sDumpFile) -{ - // Convert database DSN to psql parameters - $aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile; - - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes); - if (!is_resource($hProcess)) fail('unable to start pg_restore'); - - fclose($ahPipes[0]); - - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); - } - fclose($ahPipes[1]); - - $iReturn = proc_close($hProcess); -} - -function pgsqlRunDropAndRestore($sDumpFile) -{ - // Convert database DSN to psql parameters - $aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile; - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $aDSNInfo['hostspec']; - } - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $sCMD .= ' -U ' . $aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - $iReturn = runWithEnv($sCMD, $aProcEnv); -} - -function passthruCheckReturn($sCmd) -{ - $iResult = -1; - passthru($sCmd, $iResult); -} - -function replace_tablespace($sTemplate, $sTablespace, $sSql) -{ - if ($sTablespace) { - $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); - } else { - $sSql = str_replace($sTemplate, '', $sSql); - } - - return $sSql; -} - -function create_sql_functions($aCMDResult) -{ - global $sModulePath; - $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); - $sTemplate = str_replace('{modulepath}', $sModulePath, $sTemplate); - if ($aCMDResult['enable-diff-updates']) { - $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); - } - if ($aCMDResult['enable-debug-statements']) { - $sTemplate = str_replace('--DEBUG:', '', $sTemplate); - } - if (CONST_Limit_Reindexing) { - $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); - } - if (!CONST_Use_US_Tiger_Data) { - $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); - } - if (!CONST_Use_Aux_Location_data) { - $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); - } - pgsqlRunScript($sTemplate); -} - -function checkModulePresence() -{ - // Try accessing the C module, so we know early if something is wrong - // and can simply error out. - global $sModulePath; - $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '"; - $sSQL .= $sModulePath."/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; - $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);'; - - $oDB =& getDB(); - $oResult = $oDB->query($sSQL); - - $bResult = true; - - if (PEAR::isError($oResult)) { - echo "\nERROR: Failed to load nominatim module. Reason:\n"; - echo $oResult->userinfo."\n\n"; - $bResult = false; - } - - return $bResult; -} +didSomething($bDidSomething); diff --git a/utils/update.php b/utils/update.php index 414fadb5..fd361eb8 100755 --- a/utils/update.php +++ b/utils/update.php @@ -3,8 +3,10 @@ require_once(dirname(dirname(__FILE__)).'/settings/settings.php'); require_once(CONST_BasePath.'/lib/init-cmd.php'); +include_once(CONST_BasePath.'/lib/setup_functions.php'); ini_set('memory_limit', '800M'); + # (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) $aCMDOptions @@ -99,12 +101,7 @@ if ($aResult['init-updates']) { fail('pyosmium-get-changes not found or not usable'); } if (!$aResult['no-update-functions']) { - $sSetup ='@PHP_BIN@ '. CONST_InstallPath.'/utils/setup.php'; - $iRet = -1; - passthru($sSetup.' --create-functions --enable-diff-updates', $iRet); - if ($iRet != 0) { - fail('Error running setup script'); - } + createFunctions($aCMDResult); } $sDatabaseDate = getDatabaseDate($oDB); From c036480ce247bed75a382a230e3a109d60eb19bd Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Wed, 29 Aug 2018 21:31:19 +0200 Subject: [PATCH 02/12] first draft of setupClass --- CMakeLists.txt | 1 + lib/setup_functions.php | 1042 +++------------------------------------ utils/setup.php | 234 +++++---- utils/setupClass.php | 823 +++++++++++++++++++++++++++++++ 4 files changed, 1036 insertions(+), 1064 deletions(-) mode change 100644 => 100755 CMakeLists.txt create mode 100755 utils/setupClass.php diff --git a/CMakeLists.txt b/CMakeLists.txt old mode 100644 new mode 100755 index af6b9a92..5812883d --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,7 @@ set(CUSTOMFILES utils/query.php utils/server_compare.php utils/setup.php + utils/setupClass.php utils/specialphrases.php utils/update.php utils/warm.php diff --git a/lib/setup_functions.php b/lib/setup_functions.php index b6d927ab..da66fc7d 100755 --- a/lib/setup_functions.php +++ b/lib/setup_functions.php @@ -1,965 +1,77 @@ - $aDSNInfo['password']), $_ENV); - } - - $result = runWithEnv($sCreateDBCmd, $aProcEnv); - if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); - } - - if ($aCMDResult['setup-db'] || $aCMDResult['all']) { - info('Setup DB'); - $bDidSomething = true; - - $oDB =& getDB(); - - $fPostgresVersion = getPostgresVersion($oDB); - echo 'Postgres version found: '.$fPostgresVersion."\n"; - - if ($fPostgresVersion < 9.1) { - fail('Minimum supported version of Postgresql is 9.1.'); - } - - pgsqlRunScript($aCMDResult, 'CREATE EXTENSION IF NOT EXISTS hstore'); - pgsqlRunScript($aCMDResult, 'CREATE EXTENSION IF NOT EXISTS postgis'); - - // For extratags and namedetails the hstore_to_json converter is - // needed which is only available from Postgresql 9.3+. For older - // versions add a dummy function that returns nothing. - $iNumFunc = chksql($oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); - - if ($iNumFunc == 0) { - pgsqlRunScript($aCMDResult, "create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); - warn('Postgresql is too old. extratags and namedetails API not available.'); - } - - $fPostgisVersion = getPostgisVersion($oDB); - echo 'Postgis version found: '.$fPostgisVersion."\n"; - - if ($fPostgisVersion < 2.1) { - // Functions were renamed in 2.1 and throw an annoying deprecation warning - pgsqlRunScript($aCMDResult, 'ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint'); - pgsqlRunScript($aCMDResult, 'ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint'); - } - if ($fPostgisVersion < 2.2) { - pgsqlRunScript($aCMDResult, 'ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); - } - - $i = chksql($oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'")); - if ($i == 0) { - echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; - echo "\n createuser ".CONST_Database_Web_User."\n\n"; - exit(1); - } - - if (!checkModulePresence()) { - fail('error loading nominatim.so module'); - } - - if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { - echo 'Error: you need to download the country_osm_grid first:'; - echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; - exit(1); - } - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/country_name.sql'); - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/country_naturalearthdata.sql'); - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/country_osm_grid.sql.gz'); - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/gb_postcode_table.sql'); - - - if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/gb_postcode_data.sql.gz'); - } else { - warn('external UK postcode table not found.'); - } - - if (CONST_Use_Extra_US_Postcodes) { - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/us_postcode.sql'); - } - - if ($aCMDResult['no-partitions']) { - pgsqlRunScript($aCMDResult, 'update country_name set partition = 0'); - } - - // the following will be needed by create_functions later but - // is only defined in the subsequently called T - // Create dummies here that will be overwritten by the proper - // versions in create-tables. - pgsqlRunScript($aCMDResult, 'CREATE TABLE IF NOT EXISTS place_boundingbox ()'); - pgsqlRunScript($aCMDResult, 'CREATE TYPE wikipedia_article_match AS ()', false); - } - return $aDSNInfo; -} - -function importData($aCMDResult, $iCacheMemory, $aDSNInfo) -{ - info('Import data'); - - $osm2pgsql = CONST_Osm2pgsql_Binary; - if (!file_exists($osm2pgsql)) { - echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; - echo "Normally you should not need to set this manually.\n"; - fail("osm2pgsql not found in '$osm2pgsql'"); - } - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; - } - if (CONST_Tablespace_Osm2pgsql_Data) - $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data; - if (CONST_Tablespace_Osm2pgsql_Index) - $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index; - if (CONST_Tablespace_Place_Data) - $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data; - if (CONST_Tablespace_Place_Index) - $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; - $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; - $osm2pgsql .= ' -C '.$iCacheMemory; - $osm2pgsql .= ' -P '.$aDSNInfo['port']; - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $osm2pgsql .= ' -U ' . $aDSNInfo['username']; - } - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $osm2pgsql .= ' -H ' . $aDSNInfo['hostspec']; - } - - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - $osm2pgsql .= ' -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']; - runWithEnv($osm2pgsql, $aProcEnv); - - $oDB =& getDB(); - if (!$aCMDResult['ignore-errors'] && !chksql($oDB->getRow('select * from place limit 1'))) { - fail('No Data'); - } -} - -function createFunctions($aCMDResult) -{ - info('Create Functions'); - - if (!checkModulePresence()) { - fail('error loading nominatim.so module'); - } - - createSqlFunctions($aCMDResult); -} - -function createTables($aCMDResult) -{ - info('Create Tables'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replaceTablespace( - '{ts:address-data}', - CONST_Tablespace_Address_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:search-data}', - CONST_Tablespace_Search_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - pgsqlRunScript($aCMDResult, $sTemplate, false); - - // re-run the functions - info('Recreate Functions'); - createSqlFunctions($aCMDResult); -} - -function createPartitionTables($aCMDResult) -{ - info('Create Partition Tables'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); - $sTemplate = replaceTablespace( - '{ts:address-data}', - CONST_Tablespace_Address_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:search-data}', - CONST_Tablespace_Search_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - - pgsqlRunPartitionScript($aCMDResult, $sTemplate); -} - -function createPartitionFunctions($aCMDResult) -{ - info('Create Partition Functions'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); - - pgsqlRunPartitionScript($aCMDResult, $sTemplate); -} - -function importWikipediaArticles() -{ - $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; - $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; - if (file_exists($sWikiArticlesFile)) { - info('Importing wikipedia articles'); - pgsqlRunDropAndRestore($sWikiArticlesFile); - } else { - warn('wikipedia article dump file not found - places will have default importance'); - } - if (file_exists($sWikiRedirectsFile)) { - info('Importing wikipedia redirects'); - pgsqlRunDropAndRestore($sWikiRedirectsFile); - } else { - warn('wikipedia redirect dump file not found - some place importance values may be missing'); - } -} - -function loadData($aCMDResult, $iInstances) -{ - info('Drop old Data'); - - $oDB =& getDB(); - if (!pg_query($oDB->connection, 'TRUNCATE word')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($oDB->connection)); - echo '.'; - if (!pg_query($oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($oDB->connection)); - echo '.'; - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = chksql($oDB->getCol($sSQL)); - if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; - foreach ($aPartitions as $sPartition) { - if (!pg_query($oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($oDB->connection)); - echo '.'; - } - - // used by getorcreate_word_id to ignore frequent partial words - $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; - $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } - echo ".\n"; - - // pre-create the word list - if (!$aCMDResult['disable-token-precalc']) { - info('Loading word list'); - pgsqlRunScriptFile($aCMDResult, CONST_BasePath.'/data/words.sql'); - } - - info('Load Data'); - $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; - - $aDBInstances = array(); - $iLoadThreads = max(1, $iInstances - 1); - for ($i = 0; $i < $iLoadThreads; $i++) { - $aDBInstances[$i] =& getDB(true); - $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; - $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; - $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; - $sSQL .= ' and ST_IsValid(geometry)'; - if ($aCMDResult['verbose']) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { - fail(pg_last_error($aDBInstances[$i]->connection)); - } - } - // last thread for interpolation lines - $aDBInstances[$iLoadThreads] =& getDB(true); - $sSQL = 'insert into location_property_osmline'; - $sSQL .= ' (osm_id, address, linegeo)'; - $sSQL .= ' SELECT osm_id, address, geometry from place where '; - $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; - if ($aCMDResult['verbose']) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { - fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); - } - - $bFailed = false; - for ($i = 0; $i <= $iLoadThreads; $i++) { - while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { - $resultStatus = pg_result_status($hPGresult); - // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, - // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, - // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR - echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; - if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { - $resultError = pg_result_error($hPGresult); - echo '-- error text ' . $i . ': ' . $resultError . "\n"; - $bFailed = true; - } - } - } - if ($bFailed) { - fail('SQL errors loading placex and/or location_property_osmline tables'); - } - echo "\n"; - info('Reanalysing database'); - pgsqlRunScript($aCMDResult, 'ANALYSE'); - - $sDatabaseDate = getDatabaseDate($oDB); - pg_query($oDB->connection, 'TRUNCATE import_status'); - if ($sDatabaseDate === false) { - warn('could not determine database date.'); - } else { - $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; - pg_query($oDB->connection, $sSQL); - echo "Latest data imported from $sDatabaseDate.\n"; - } -} - -function importTigerData($aCMDResult, $iInstances) -{ - info('Import Tiger data'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - pgsqlRunScript($aCMDResult, $sTemplate, false); - - $aDBInstances = array(); - for ($i = 0; $i < $iInstances; $i++) { - $aDBInstances[$i] =& getDB(true); - } - - foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) { - echo $sFile.': '; - $hFile = fopen($sFile, 'r'); - $sSQL = fgets($hFile, 100000); - $iLines = 0; - - while (true) { - for ($i = 0; $i < $iInstances; $i++) { - if (!pg_connection_busy($aDBInstances[$i]->connection)) { - while (pg_get_result($aDBInstances[$i]->connection)); - $sSQL = fgets($hFile, 100000); - if (!$sSQL) break 2; - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($oDB->connection)); - $iLines++; - if ($iLines == 1000) { - echo '.'; - $iLines = 0; - } - } - } - usleep(10); - } - - fclose($hFile); - - $bAnyBusy = true; - while ($bAnyBusy) { - $bAnyBusy = false; - for ($i = 0; $i < $iInstances; $i++) { - if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; - } - usleep(10); - } - echo "\n"; - } - - info('Creating indexes on Tiger data'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - pgsqlRunScript($aCMDResult, $sTemplate, false); -} - -function calculatePostcodes($aCMDResult) -{ - info('Calculate Postcodes'); - $oDB =& getDB(); - if (!pg_query($oDB->connection, 'TRUNCATE location_postcode')) { - fail(pg_last_error($oDB->connection)); - } - - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; - $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; - $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; - $sSQL .= ' FROM placex'; - $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; - $sSQL .= ' AND geometry IS NOT null'; - $sSQL .= ' GROUP BY country_code, pc'; - - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } - - if (CONST_Use_Extra_US_Postcodes) { - // only add postcodes that are not yet available in OSM - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; - $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; - $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'us')"; - if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); - } - - // add missing postcodes for GB (if available) - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; - $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'gb')"; - if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); - - if (!$aCMDResult['all']) { - $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; - $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } - } - $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; - $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; - - if (!pg_query($oDB->connection, $sSQL)) { - fail(pg_last_error($oDB->connection)); - } -} - -function index($aCMDResult, $aDSNInfo, $iInstances) -{ - $sOutputFile = ''; - $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$iInstances.$sOutputFile; - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $sBaseCmd .= ' -H ' . $aDSNInfo['hostspec']; - } - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $sBaseCmd .= ' -U ' . $aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - info('Index ranks 0 - 4'); - $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$aCMDResult['index-noanalyse']) pgsqlRunScript($aCMDResult, 'ANALYSE'); - info('Index ranks 5 - 25'); - $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$aCMDResult['index-noanalyse']) pgsqlRunScript($aCMDResult, 'ANALYSE'); - info('Index ranks 26 - 30'); - $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - - info('Index postcodes'); - $oDB =& getDB(); - $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; - if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); -} - -function createSearchIndices($aCMDResult) -{ - info('Create Search indices'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - - pgsqlRunScript($aCMDResult, $sTemplate); -} - -function createCountryNames($aCMDResult) -{ - info('Create search index for default country names'); - - pgsqlRunScript($aCMDResult, "select getorcreate_country(make_standard_name('uk'), 'gb')"); - pgsqlRunScript($aCMDResult, "select getorcreate_country(make_standard_name('united states'), 'us')"); - pgsqlRunScript($aCMDResult, 'select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); - pgsqlRunScript($aCMDResult, "select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); - - $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v), country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; - if (CONST_Languages) { - $sSQL .= 'in '; - $sDelim = '('; - foreach (explode(',', CONST_Languages) as $sLang) { - $sSQL .= $sDelim."'name:$sLang'"; - $sDelim = ','; - } - $sSQL .= ')'; - } else { - // all include all simple name tags - $sSQL .= "like 'name:%'"; - } - $sSQL .= ') v'; - pgsqlRunScript($aCMDResult, $sSQL); -} - -function drop($aCMDResult) -{ - info('Drop tables only required for updates'); - // The implementation is potentially a bit dangerous because it uses - // a positive selection of tables to keep, and deletes everything else. - // Including any tables that the unsuspecting user might have manually - // created. USE AT YOUR OWN PERIL. - - - // tables we want to keep. everything else goes. - $aKeepTables = array( - '*columns', - 'import_polygon_*', - 'import_status', - 'place_addressline', - 'location_postcode', - 'location_property*', - 'placex', - 'search_name', - 'seq_*', - 'word', - 'query_log', - 'new_query_log', - 'spatial_ref_sys', - 'country_name', - 'place_classtype_*' - ); - - $oDB =& getDB(); - $aDropTables = array(); - $aHaveTables = chksql($oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); - - foreach ($aHaveTables as $sTable) { - $bFound = false; - foreach ($aKeepTables as $sKeep) { - if (fnmatch($sKeep, $sTable)) { - $bFound = true; - break; - } - } - if (!$bFound) array_push($aDropTables, $sTable); - } - - foreach ($aDropTables as $sDrop) { - if ($aCMDResult['verbose']) echo "dropping table $sDrop\n"; - @pg_query($oDB->connection, "DROP TABLE $sDrop CASCADE"); - // ignore warnings/errors as they might be caused by a table having - // been deleted already by CASCADE - } - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - if ($aCMDResult['verbose']) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; - unlink(CONST_Osm2pgsql_Flatnode_File); - } -} - -function didSomething($bDidSomething) -{ - if (!$bDidSomething) { - showUsage($aCMDOptions, true); - } else { - echo "Summary of warnings:\n\n"; - repeatWarnings(); - echo "\n"; - info('Setup finished.'); - } -} - -// ********************************* - -function pgsqlRunScriptFile($aCMDResult, $sFilename) -{ - if (!file_exists($sFilename)) fail('unable to find '.$sFilename); - // Convert database DSN to psql parameters - $aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - $sCMD = 'psql -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database']; - if (!$aCMDResult['verbose']) { - $sCMD .= ' -q'; - } - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $aDSNInfo['hostspec']; - } - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $sCMD .= ' -U ' . $aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - $ahGzipPipes = null; - if (preg_match('/\\.gz$/', $sFilename)) { - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); - if (!is_resource($hGzipProcess)) fail('unable to start zcat'); - $aReadPipe = $ahGzipPipes[1]; - fclose($ahGzipPipes[0]); - } else { - $sCMD .= ' -f '.$sFilename; - $aReadPipe = array('pipe', 'r'); - } - $aDescriptors = array( - 0 => $aReadPipe, - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); - if (!is_resource($hProcess)) fail('unable to start pgsql'); - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); - } - fclose($ahPipes[1]); - $iReturn = proc_close($hProcess); - if ($iReturn > 0) { - fail("pgsql returned with error code ($iReturn)"); - } - if ($ahGzipPipes) { - fclose($ahGzipPipes[1]); - proc_close($hGzipProcess); - } -} - -function pgsqlRunScript($aCMDResult, $sScript, $bfatal = true) -{ - runSQLScript( - $sScript, - $bfatal, - $aCMDResult['verbose'], - $aCMDResult['ignore-errors'] - ); -} - -function pgsqlRunPartitionScript($aCMDResult, $sTemplate) -{ - global $aCMDResult; - $oDB =& getDB(); - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = chksql($oDB->getCol($sSQL)); - if (!$aCMDResult['no-partitions']) $aPartitions[] = 0; - - preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); - foreach ($aMatches as $aMatch) { - $sResult = ''; - foreach ($aPartitions as $sPartitionName) { - $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); - } - $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); - } - - pgsqlRunScript($aCMDResult, $sTemplate); -} - -function pgsqlRunRestoreData($sDumpFile) -{ - // Convert database DSN to psql parameters - $aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc -a '.$sDumpFile; - - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes); - if (!is_resource($hProcess)) fail('unable to start pg_restore'); - - fclose($ahPipes[0]); - - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); - } - fclose($ahPipes[1]); - - $iReturn = proc_close($hProcess); -} - -function pgsqlRunDropAndRestore($sDumpFile) -{ - // Convert database DSN to psql parameters - $aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; - $sCMD = 'pg_restore -p '.$aDSNInfo['port'].' -d '.$aDSNInfo['database'].' -Fc --clean '.$sDumpFile; - if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $aDSNInfo['hostspec']; - } - if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $sCMD .= ' -U ' . $aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $aDSNInfo['password']), $_ENV); - } - - $iReturn = runWithEnv($sCMD, $aProcEnv); -} - -function passthpassthruCheckReturn($sCmd) -{ - $iResult = -1; - passthru($sCmd, $iResult); -} - -function replaceTablespace($sTemplate, $sTablespace, $sSql) -{ - if ($sTablespace) { - $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); - } else { - $sSql = str_replace($sTemplate, '', $sSql); - } - - return $sSql; -} - -function createSqlFunctions($aCMDResult) -{ - $sModulePath = CONST_Database_Module_Path; - $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); - $sTemplate = str_replace('{modulepath}', $sModulePath, $sTemplate); - if ($aCMDResult['enable-diff-updates']) { - $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); - } - if ($aCMDResult['enable-debug-statements']) { - $sTemplate = str_replace('--DEBUG:', '', $sTemplate); - } - if (CONST_Limit_Reindexing) { - $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); - } - if (!CONST_Use_US_Tiger_Data) { - $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); - } - if (!CONST_Use_Aux_Location_data) { - $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); - } - pgsqlRunScript($aCMDResult, $sTemplate); -} - -function checkModulePresence() -{ - // Try accessing the C module, so we know early if something is wrong - // and can simply error out. - $sModulePath = CONST_Database_Module_Path; - $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '"; - $sSQL .= $sModulePath."/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; - $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);'; - - $oDB =& getDB(); - $oResult = $oDB->query($sSQL); - - $bResult = true; - - if (PEAR::isError($oResult)) { - echo "\nERROR: Failed to load nominatim module. Reason:\n"; - echo $oResult->userinfo."\n\n"; - $bResult = false; - } - - return $bResult; -} - -function createSetupArgvArray() -{ - $aCMDOptions - = array( - 'Create and setup nominatim search system', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - - array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), - array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), - - array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), - - array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), - array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), - array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), - array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), - array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), - array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), - array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), - array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), - array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), - array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), - array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), - array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), - array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), - array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), - array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), - array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), - array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), - array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), - array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), - array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), - ); - return $aCMDOptions; -} +query($sSQL); + + $bResult = true; + + if (PEAR::isError($oResult)) { + echo "\nERROR: Failed to load nominatim module. Reason:\n"; + echo $oResult->userinfo."\n\n"; + $bResult = false; + } + + return $bResult; +} + +function createSetupArgvArray() { + $aCMDOptions + = array( + 'Create and setup nominatim search system', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), + array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), + + array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), + + array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), + array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), + array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), + array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), + array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), + array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), + array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), + array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), + array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), + array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), + array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), + array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), + array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), + array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), + array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), + array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), + array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), + array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), + array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), + array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), + ); + return $aCMDOptions; +} diff --git a/utils/setup.php b/utils/setup.php index 449dd4da..6afe196c 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -1,99 +1,135 @@ -#!@PHP_BIN@ -Cq -indirect via init-cmd.php -> /lib/cmd.php for runWithEnv, getCmdOpt +// ->indirect via init-cmd.php -> /lib/init.php -> db.php for &getDB() + +include_once(CONST_BasePath.'/lib/setup_functions.php'); +include_once(CONST_BasePath.'/lib/setup_functions.php'); +ini_set('memory_limit', '800M'); + + +$aCMDOptions = createSetupArgvArray(); + +// $aCMDOptions passed to getCmdOpt by reference +getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); + + +$bDidSomething = false; + +//******************************************************* +// Making some sanity check: +// Check if osm-file is set and points to a valid file +if ($aCMDResult['import-data'] || $aCMDResult['all']) { + // to remain in /lib/setup_functions.php function + checkInFile($aCMDResult['osm-file']); + echo $aCMDResult['osm-file']; +} + +// osmosis init is no longer supported +if ($aCMDResult['osmosis-init']) { + $bDidSomething = true; + echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n"; +} + +// ****************************************************** +// instantiate Setup class +$cSetup = new SetupFunctions ($aCMDResult); +if ($aCMDResult['create-db'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createDB(); +} + +// ******************************************************* +// go through complete process if 'all' is selected or start selected functions +if ($aCMDResult['setup-db'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> setupDB(); +} + +// Try accessing the C module, so we know early if something is wrong +if (!checkModulePresence()) { + fail('error loading nominatim.so module'); +} + +if ($aCMDResult['import-data'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> importData($aCMDResult['osm-file']); +} + +if ($aCMDResult['create-functions'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createFunctions(); +} + +if ($aCMDResult['create-tables'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createTables(); +} + +if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createPartitionTables(); +} + +if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createPartitionFunctions(); +} +/* +if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> importWikipediaArticles(); +} +*/ +if ($aCMDResult['load-data'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> loadData($aCMDResult['disable-token-precalc']); +} + +if ($aCMDResult['import-tiger-data']) { + $bDidSomething = true; + $cSetup -> importTigerData(); +} + +if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> calculatePostcodes($aCMDResult['all']); +} + +if ($aCMDResult['index'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> index($aCMDResult['index-noanalyse']); +} + +if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createSearchIndices(); +} + +if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup -> createCountryNames($aCMDResult); +} + +if ($aCMDResult['drop']) { + $bDidSomething = true; + $cSetup -> drop($aCMDResult); +} + +// ****************************************************** +// If we did something, repeat the warnings +if (!$bDidSomething) { + showUsage($aCMDOptions, true); +} else { + echo "Summary of warnings:\n\n"; + repeatWarnings(); + echo "\n"; + info('Setup finished.'); +} + + + diff --git a/utils/setupClass.php b/utils/setupClass.php new file mode 100755 index 00000000..03a0205b --- /dev/null +++ b/utils/setupClass.php @@ -0,0 +1,823 @@ +iInstances = isset($aCMDResult['threads']) + ? $aCMDResult['threads'] + : (min(16, getProcessorCount()) - 1); + + + if ($this->iInstances < 1) { + $this->iInstances = 1; + warn('resetting threads to '.$this->iInstances); + } + + // Assume we can steal all the cache memory in the box (unless told otherwise) + if (isset($aCMDResult['osm2pgsql-cache'])) { + $this->iCacheMemory = $aCMDResult['osm2pgsql-cache']; + } else { + $this->iCacheMemory = getCacheMemoryMB(); + } + + $this->sModulePath = CONST_Database_Module_Path; + info('module path: ' . $this->sModulePath); + + // prepares DB for import or update, sets the Data Source Name + $this->aDSNInfo = DB::parseDSN(CONST_Database_DSN); + if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; + + // setting member variables based on command line options stored in $aCMDResult + $this->sVerbose = $aCMDResult['verbose']; + $this->sIgnoreErrors = $aCMDResult['ignore-errors']; + $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; + $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; + $this->bNoPartitions = $aCMDResult['no-partitions']; + + } + + public function createDB(){ + info('Create DB'); + $sDB = DB::connect(CONST_Database_DSN, false); + if (!PEAR::isError($sDB)) { + fail('database already exists ('.CONST_Database_DSN.')'); + } + + $sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database']; + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username']; + } + + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCreateDBCmd .= ' -h ' . $this->aDSNInfo['hostspec']; + } + + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + + $result = runWithEnv($sCreateDBCmd, $aProcEnv); + if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); + } + + public function setupDB() { + info('Setup DB'); + $this->oDB =& getDB(); + + $fPostgresVersion = getPostgresVersion($this->oDB); + echo 'Postgres version found: '.$fPostgresVersion."\n"; + + if ($fPostgresVersion < 9.1) { + fail('Minimum supported version of Postgresql is 9.1.'); + } + + $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore'); + $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); + + // For extratags and namedetails the hstore_to_json converter is + // needed which is only available from Postgresql 9.3+. For older + // versions add a dummy function that returns nothing. + $iNumFunc = chksql($this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); + + if ($iNumFunc == 0) { + $this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); + warn('Postgresql is too old. extratags and namedetails API not available.'); + } + + + $fPostgisVersion = getPostgisVersion($this->oDB); + echo 'Postgis version found: '.$fPostgisVersion."\n"; + + if ($fPostgisVersion < 2.1) { + // Functions were renamed in 2.1 and throw an annoying deprecation warning + $this->pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint'); + $this->pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint'); + } + if ($fPostgisVersion < 2.2) { + $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); + } + + $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'")); + if ($i == 0) { + echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; + echo "\n createuser ".CONST_Database_Web_User."\n\n"; + exit(1); + } + + if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { + echo 'Error: you need to download the country_osm_grid first:'; + echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; + exit(1); + } + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql'); + + + if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz'); + } else { + warn('external UK postcode table not found.'); + } + + if (CONST_Use_Extra_US_Postcodes) { + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); + } + + if ($this->bNoPartitions) { + $this->pgsqlRunScript('update country_name set partition = 0'); + } + + // the following will be needed by create_functions later but + // is only defined in the subsequently called T + // Create dummies here that will be overwritten by the proper + // versions in create-tables. + $this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); + $this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); + } + + public function importData($sOSMFile) { + info('Import data'); + + $osm2pgsql = CONST_Osm2pgsql_Binary; + if (!file_exists($osm2pgsql)) { + echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; + echo "Normally you should not need to set this manually.\n"; + fail("osm2pgsql not found in '$osm2pgsql'"); + } + + + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { + $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; + } + + if (CONST_Tablespace_Osm2pgsql_Data) + $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data; + if (CONST_Tablespace_Osm2pgsql_Index) + $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index; + if (CONST_Tablespace_Place_Data) + $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data; + if (CONST_Tablespace_Place_Index) + $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; + $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; + $osm2pgsql .= ' -C '.$this->iCacheMemory; + $osm2pgsql .= ' -P '.$this->aDSNInfo['port']; + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $osm2pgsql .= ' -U ' . $this->aDSNInfo['username']; + } + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $osm2pgsql .= ' -H ' . $this->aDSNInfo['hostspec']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $osm2pgsql .= ' -d '.$this->aDSNInfo['database'].' '.$sOSMFile; + runWithEnv($osm2pgsql, $aProcEnv); + if ($this->oDB == null) $this->oDB =& getDB(); + if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) { + fail('No Data'); + } + } + + public function createFunctions() { + info('Create Functions'); + + $this->createSqlFunctions(); + } + + public function createTables() { + info('Create Tables'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:address-data}', + CONST_Tablespace_Address_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-data}', + CONST_Tablespace_Search_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + + $this->pgsqlRunScript($sTemplate, false); + + // re-run the functions + info('Recreate Functions'); + $this->createSqlFunctions(); + } + + public function createPartitionTables() { + info('Create Partition Tables'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); + $sTemplate = $this->replaceTablespace( + '{ts:address-data}', + CONST_Tablespace_Address_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:search-data}', + CONST_Tablespace_Search_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + + $this->pgsqlRunPartitionScript($sTemplate); + } + + public function createPartitionFunctions() { + info('Create Partition Functions'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); + $this->pgsqlRunPartitionScript($sTemplate); + } + + public function importWikipediaArticles() { + $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; + $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; + if (file_exists($sWikiArticlesFile)) { + info('Importing wikipedia articles'); + $this->pgsqlRunDropAndRestore($sWikiArticlesFile); + } else { + warn('wikipedia article dump file not found - places will have default importance'); + } + if (file_exists($sWikiRedirectsFile)) { + info('Importing wikipedia redirects'); + $this->pgsqlRunDropAndRestore($sWikiRedirectsFile); + } else { + warn('wikipedia redirect dump file not found - some place importance values may be missing'); + } + echo ' finish wikipedia'; + } + + public function loadData($bDisableTokenPrecalc) { + info('Drop old Data'); + + if ($this->oDB == null) $this->oDB =& getDB(); + + if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection)); + echo '.'; + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($this->oDB->getCol($sSQL)); + if (!$this->bNoPartitions) $aPartitions[] = 0; + foreach ($aPartitions as $sPartition) { + if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection)); + echo '.'; + } + + // used by getorcreate_word_id to ignore frequent partial words + $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; + $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + echo ".\n"; + + // pre-create the word list + if (!$bDisableTokenPrecalc) { + info('Loading word list'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql'); + } + + info('Load Data'); + $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; + $aDBInstances = array(); + $iLoadThreads = max(1, $this->iInstances - 1); + for ($i = 0; $i < $iLoadThreads; $i++) { + $aDBInstances[$i] =& getDB(true); + $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; + $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; + $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; + $sSQL .= ' and ST_IsValid(geometry)'; + if ($this->sVerbose) echo "$sSQL\n"; + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { + fail(pg_last_error($aDBInstances[$i]->connection)); + } + } + + // last thread for interpolation lines + + $aDBInstances[$iLoadThreads] =& getDB(true); + $sSQL = 'insert into location_property_osmline'; + $sSQL .= ' (osm_id, address, linegeo)'; + $sSQL .= ' SELECT osm_id, address, geometry from place where '; + $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; + if ($this->sVerbose) echo "$sSQL\n"; + if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { + fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); + } + + $bFailed = false; + for ($i = 0; $i <= $iLoadThreads; $i++) { + while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { + $resultStatus = pg_result_status($hPGresult); + // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, + // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, + // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR + echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; + if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { + $resultError = pg_result_error($hPGresult); + echo '-- error text ' . $i . ': ' . $resultError . "\n"; + $bFailed = true; + } + } + } + if ($bFailed) { + fail('SQL errors loading placex and/or location_property_osmline tables'); + } + echo "\n"; + info('Reanalysing database'); + $this->pgsqlRunScript('ANALYSE'); + + $sDatabaseDate = getDatabaseDate($this->oDB); + pg_query($this->oDB->connection, 'TRUNCATE import_status'); + if ($sDatabaseDate === false) { + warn('could not determine database date.'); + } else { + $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; + pg_query($this->oDB->connection, $sSQL); + echo "Latest data imported from $sDatabaseDate.\n"; + } + } + + public function importTigerData() { + info('Import Tiger data'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate, false); + + $aDBInstances = array(); + for ($i = 0; $i < $this->iInstances; $i++) { + $aDBInstances[$i] =& getDB(true); + } + + foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) { + echo $sFile.': '; + $hFile = fopen($sFile, 'r'); + $sSQL = fgets($hFile, 100000); + $iLines = 0; + while (true) { + for ($i = 0; $i < $this->iInstances; $i++) { + if (!pg_connection_busy($aDBInstances[$i]->connection)) { + while (pg_get_result($aDBInstances[$i]->connection)); + $sSQL = fgets($hFile, 100000); + if (!$sSQL) break 2; + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); + $iLines++; + if ($iLines == 1000) { + echo '.'; + $iLines = 0; + } + } + } + usleep(10); + } + fclose($hFile); + + $bAnyBusy = true; + while ($bAnyBusy) { + $bAnyBusy = false; + for ($i = 0; $i < $this->iInstances; $i++) { + if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; + } + usleep(10); + } + echo "\n"; + } + + info('Creating indexes on Tiger data'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate, false); + } + + public function calculatePostcodes($bCMDResultAll) { + info('Calculate Postcodes'); + if ($this->oDB == null) $this->oDB =& getDB(); + if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { + fail(pg_last_error($this->oDB->connection)); + } + + + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; + $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; + $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; + $sSQL .= ' FROM placex'; + $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; + $sSQL .= ' AND geometry IS NOT null'; + $sSQL .= ' GROUP BY country_code, pc'; + + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + if (CONST_Use_Extra_US_Postcodes) { + // only add postcodes that are not yet available in OSM + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; + $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; + $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; + $sSQL .= ' (SELECT postcode FROM location_postcode'; + $sSQL .= " WHERE country_code = 'us')"; + if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); + } + + // add missing postcodes for GB (if available) + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; + $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; + $sSQL .= ' (SELECT postcode FROM location_postcode'; + $sSQL .= " WHERE country_code = 'gb')"; + if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); + + if (!$bCMDResultAll) { + $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; + $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + } + $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; + $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; + + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + } + + public function index($bIndexNoanalyse) { + $sOutputFile = ''; + $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P '.$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile; + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sBaseCmd .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + + info('Index ranks 0 - 4'); + $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); + info('Index ranks 5 - 25'); + $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); + info('Index ranks 26 - 30'); + $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + info('Index postcodes'); + if ($this->oDB == null) $this->oDB =& getDB(); + $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; + if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); + } + + public function createSearchIndices() { + info('Create Search indices'); + + $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate); + } + + public function createCountryNames() { + info('Create search index for default country names'); + + $this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); + $this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); + $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); + $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); + $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v), country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; + if (CONST_Languages) { + $sSQL .= 'in '; + $sDelim = '('; + foreach (explode(',', CONST_Languages) as $sLang) { + $sSQL .= $sDelim."'name:$sLang'"; + $sDelim = ','; + } + $sSQL .= ')'; + } else { + // all include all simple name tags + $sSQL .= "like 'name:%'"; + } + $sSQL .= ') v'; + $this->pgsqlRunScript($sSQL); + } + + public function drop() { + info('Drop tables only required for updates'); + + // The implementation is potentially a bit dangerous because it uses + // a positive selection of tables to keep, and deletes everything else. + // Including any tables that the unsuspecting user might have manually + // created. USE AT YOUR OWN PERIL. + // tables we want to keep. everything else goes. + $aKeepTables = array( + '*columns', + 'import_polygon_*', + 'import_status', + 'place_addressline', + 'location_postcode', + 'location_property*', + 'placex', + 'search_name', + 'seq_*', + 'word', + 'query_log', + 'new_query_log', + 'spatial_ref_sys', + 'country_name', + 'place_classtype_*' + ); + + if ($this->oDB = null) $this->oDB =& getDB(); + $aDropTables = array(); + $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); + + foreach ($aHaveTables as $sTable) { + $bFound = false; + foreach ($aKeepTables as $sKeep) { + if (fnmatch($sKeep, $sTable)) { + $bFound = true; + break; + } + } + if (!$bFound) array_push($aDropTables, $sTable); + } + foreach ($aDropTables as $sDrop) { + if ($this->sVerbose) echo "dropping table $sDrop\n"; + @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE"); + // ignore warnings/errors as they might be caused by a table having + // been deleted already by CASCADE + } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { + if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; + unlink(CONST_Osm2pgsql_Flatnode_File); + } + } + + private function pgsqlRunDropAndRestore($sDumpFile) { + if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; + $sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile; + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCMD .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" + } + + private function pgsqlRunScript($sScript, $bfatal = true) { + runSQLScript( + $sScript, + $bfatal, + $this->sVerbose, + $this->sIgnoreErrors + ); + } + + private function createSqlFunctions() { + $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); + $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); + if ($this->bEnableDiffUpdates) { + $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); + } + if ($this->bEnableDebugStatements) { + $sTemplate = str_replace('--DEBUG:', '', $sTemplate); + } + if (CONST_Limit_Reindexing) { + $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); + } + if (!CONST_Use_US_Tiger_Data) { + $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); + } + if (!CONST_Use_Aux_Location_data) { + $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); + } + $this->pgsqlRunScript($sTemplate); + } + + private function pgsqlRunPartitionScript($sTemplate) { + if ($this->oDB == null) $this->oDB =& getDB(); + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($this->oDB->getCol($sSQL)); + if (!$this->bNoPartitions) $aPartitions[] = 0; + + preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); + foreach ($aMatches as $aMatch) { + $sResult = ''; + foreach ($aPartitions as $sPartitionName) { + $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); + } + $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); + } + + $this->pgsqlRunScript($sTemplate); + } + + private function pgsqlRunScriptFile($sFilename) { + if (!file_exists($sFilename)) fail('unable to find '.$sFilename); + + $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database']; + if (!$this->sVerbose) { + $sCMD .= ' -q'; + } + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCMD .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $ahGzipPipes = null; + if (preg_match('/\\.gz$/', $sFilename)) { + $aDescriptors = array( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a') + ); + $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); + if (!is_resource($hGzipProcess)) fail('unable to start zcat'); + $aReadPipe = $ahGzipPipes[1]; + fclose($ahGzipPipes[0]); + } else { + $sCMD .= ' -f '.$sFilename; + $aReadPipe = array('pipe', 'r'); + } + $aDescriptors = array( + 0 => $aReadPipe, + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a') + ); + $ahPipes = null; + $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); + if (!is_resource($hProcess)) fail('unable to start pgsql'); + // TODO: error checking + while (!feof($ahPipes[1])) { + echo fread($ahPipes[1], 4096); + } + fclose($ahPipes[1]); + $iReturn = proc_close($hProcess); + if ($iReturn > 0) { + fail("pgsql returned with error code ($iReturn)"); + } + if ($ahGzipPipes) { + fclose($ahGzipPipes[1]); + proc_close($hGzipProcess); + } + } + private function replaceTablespace($sTemplate, $sTablespace, $sSql) { + if ($sTablespace) { + $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); + } else { + $sSql = str_replace($sTemplate, '', $sSql); + } + return $sSql; + } +} From b2f3cfde0bb3d6a02e248272c3101970d7cfeb2e Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Wed, 29 Aug 2018 22:54:28 +0200 Subject: [PATCH 03/12] splitted createTables and changed formatting to please Travis --- lib/setup_functions.php | 9 +++-- utils/setup.php | 12 +++--- utils/setupClass.php | 85 +++++++++++++++++++++++++---------------- 3 files changed, 64 insertions(+), 42 deletions(-) diff --git a/lib/setup_functions.php b/lib/setup_functions.php index da66fc7d..fec28f17 100755 --- a/lib/setup_functions.php +++ b/lib/setup_functions.php @@ -1,6 +1,7 @@ createDB(); @@ -68,6 +68,7 @@ if ($aCMDResult['create-functions'] || $aCMDResult['all']) { if ($aCMDResult['create-tables'] || $aCMDResult['all']) { $bDidSomething = true; $cSetup -> createTables(); + $cSetup -> recreateFunction(); } if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { @@ -79,12 +80,12 @@ if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { $bDidSomething = true; $cSetup -> createPartitionFunctions(); } -/* + if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { $bDidSomething = true; $cSetup -> importWikipediaArticles(); } -*/ + if ($aCMDResult['load-data'] || $aCMDResult['all']) { $bDidSomething = true; $cSetup -> loadData($aCMDResult['disable-token-precalc']); @@ -130,6 +131,3 @@ if (!$bDidSomething) { echo "\n"; info('Setup finished.'); } - - - diff --git a/utils/setupClass.php b/utils/setupClass.php index 03a0205b..c5161542 100755 --- a/utils/setupClass.php +++ b/utils/setupClass.php @@ -1,9 +1,6 @@ iInstances = isset($aCMDResult['threads']) @@ -50,10 +46,10 @@ class SetupFunctions $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; $this->bNoPartitions = $aCMDResult['no-partitions']; - } - public function createDB(){ + public function createDB() + { info('Create DB'); $sDB = DB::connect(CONST_Database_DSN, false); if (!PEAR::isError($sDB)) { @@ -78,7 +74,8 @@ class SetupFunctions if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); } - public function setupDB() { + public function setupDB() + { info('Setup DB'); $this->oDB =& getDB(); @@ -155,7 +152,8 @@ class SetupFunctions $this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); } - public function importData($sOSMFile) { + public function importData($sOSMFile) + { info('Import data'); $osm2pgsql = CONST_Osm2pgsql_Binary; @@ -200,13 +198,15 @@ class SetupFunctions } } - public function createFunctions() { + public function createFunctions() + { info('Create Functions'); $this->createSqlFunctions(); } - public function createTables() { + public function createTables() + { info('Create Tables'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); @@ -243,13 +243,17 @@ class SetupFunctions ); $this->pgsqlRunScript($sTemplate, false); + } + public function recreateFunction() + { // re-run the functions info('Recreate Functions'); $this->createSqlFunctions(); } - public function createPartitionTables() { + public function createPartitionTables() + { info('Create Partition Tables'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); @@ -292,14 +296,16 @@ class SetupFunctions $this->pgsqlRunPartitionScript($sTemplate); } - public function createPartitionFunctions() { + public function createPartitionFunctions() + { info('Create Partition Functions'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); $this->pgsqlRunPartitionScript($sTemplate); } - public function importWikipediaArticles() { + public function importWikipediaArticles() + { $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; if (file_exists($sWikiArticlesFile)) { @@ -315,9 +321,10 @@ class SetupFunctions warn('wikipedia redirect dump file not found - some place importance values may be missing'); } echo ' finish wikipedia'; - } + } - public function loadData($bDisableTokenPrecalc) { + public function loadData($bDisableTokenPrecalc) + { info('Drop old Data'); if ($this->oDB == null) $this->oDB =& getDB(); @@ -382,7 +389,6 @@ class SetupFunctions } // last thread for interpolation lines - $aDBInstances[$iLoadThreads] =& getDB(true); $sSQL = 'insert into location_property_osmline'; $sSQL .= ' (osm_id, address, linegeo)'; @@ -426,7 +432,8 @@ class SetupFunctions } } - public function importTigerData() { + public function importTigerData() + { info('Import Tiger data'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); @@ -498,7 +505,8 @@ class SetupFunctions $this->pgsqlRunScript($sTemplate, false); } - public function calculatePostcodes($bCMDResultAll) { + public function calculatePostcodes($bCMDResultAll) + { info('Calculate Postcodes'); if ($this->oDB == null) $this->oDB =& getDB(); if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { @@ -556,9 +564,11 @@ class SetupFunctions } } - public function index($bIndexNoanalyse) { + public function index($bIndexNoanalyse) + { $sOutputFile = ''; - $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P '.$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile; + $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P ' + .$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile; if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; } @@ -591,9 +601,10 @@ class SetupFunctions if ($this->oDB == null) $this->oDB =& getDB(); $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); - } + } - public function createSearchIndices() { + public function createSearchIndices() + { info('Create Search indices'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); @@ -616,14 +627,16 @@ class SetupFunctions $this->pgsqlRunScript($sTemplate); } - public function createCountryNames() { + public function createCountryNames() + { info('Create search index for default country names'); $this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); $this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); - $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v), country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; + $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),' + .'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; if (CONST_Languages) { $sSQL .= 'in '; $sDelim = '('; @@ -640,7 +653,8 @@ class SetupFunctions $this->pgsqlRunScript($sSQL); } - public function drop() { + public function drop() + { info('Drop tables only required for updates'); // The implementation is potentially a bit dangerous because it uses @@ -693,7 +707,8 @@ class SetupFunctions } } - private function pgsqlRunDropAndRestore($sDumpFile) { + private function pgsqlRunDropAndRestore($sDumpFile) + { if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; $sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile; if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { @@ -709,7 +724,8 @@ class SetupFunctions $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" } - private function pgsqlRunScript($sScript, $bfatal = true) { + private function pgsqlRunScript($sScript, $bfatal = true) + { runSQLScript( $sScript, $bfatal, @@ -718,7 +734,8 @@ class SetupFunctions ); } - private function createSqlFunctions() { + private function createSqlFunctions() + { $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); if ($this->bEnableDiffUpdates) { @@ -739,7 +756,8 @@ class SetupFunctions $this->pgsqlRunScript($sTemplate); } - private function pgsqlRunPartitionScript($sTemplate) { + private function pgsqlRunPartitionScript($sTemplate) + { if ($this->oDB == null) $this->oDB =& getDB(); $sSQL = 'select distinct partition from country_name'; @@ -756,9 +774,10 @@ class SetupFunctions } $this->pgsqlRunScript($sTemplate); - } + } - private function pgsqlRunScriptFile($sFilename) { + private function pgsqlRunScriptFile($sFilename) + { if (!file_exists($sFilename)) fail('unable to find '.$sFilename); $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database']; @@ -812,7 +831,9 @@ class SetupFunctions proc_close($hGzipProcess); } } - private function replaceTablespace($sTemplate, $sTablespace, $sSql) { + + private function replaceTablespace($sTemplate, $sTablespace, $sSql) + { if ($sTablespace) { $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); } else { From a3b4f80c996056136828560e8e5fb26db367f238 Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Fri, 31 Aug 2018 21:31:38 +0200 Subject: [PATCH 04/12] small fixes on setup.php and a bring update.php to work --- CMakeLists.txt | 1 - utils/setupClass.php => lib/SetupClass.php | 1794 +++++++++++--------- lib/setup_functions.php | 206 ++- utils/setup.php | 266 +-- utils/update.php | 190 +-- 5 files changed, 1297 insertions(+), 1160 deletions(-) rename utils/setupClass.php => lib/SetupClass.php (72%) mode change 100755 => 100644 diff --git a/CMakeLists.txt b/CMakeLists.txt index 5812883d..af6b9a92 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,6 @@ set(CUSTOMFILES utils/query.php utils/server_compare.php utils/setup.php - utils/setupClass.php utils/specialphrases.php utils/update.php utils/warm.php diff --git a/utils/setupClass.php b/lib/SetupClass.php old mode 100755 new mode 100644 similarity index 72% rename from utils/setupClass.php rename to lib/SetupClass.php index c5161542..d50d9905 --- a/utils/setupClass.php +++ b/lib/SetupClass.php @@ -1,844 +1,950 @@ -iInstances = isset($aCMDResult['threads']) - ? $aCMDResult['threads'] - : (min(16, getProcessorCount()) - 1); - - - if ($this->iInstances < 1) { - $this->iInstances = 1; - warn('resetting threads to '.$this->iInstances); - } - - // Assume we can steal all the cache memory in the box (unless told otherwise) - if (isset($aCMDResult['osm2pgsql-cache'])) { - $this->iCacheMemory = $aCMDResult['osm2pgsql-cache']; - } else { - $this->iCacheMemory = getCacheMemoryMB(); - } - - $this->sModulePath = CONST_Database_Module_Path; - info('module path: ' . $this->sModulePath); - - // prepares DB for import or update, sets the Data Source Name - $this->aDSNInfo = DB::parseDSN(CONST_Database_DSN); - if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; - - // setting member variables based on command line options stored in $aCMDResult - $this->sVerbose = $aCMDResult['verbose']; - $this->sIgnoreErrors = $aCMDResult['ignore-errors']; - $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; - $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; - $this->bNoPartitions = $aCMDResult['no-partitions']; - } - - public function createDB() - { - info('Create DB'); - $sDB = DB::connect(CONST_Database_DSN, false); - if (!PEAR::isError($sDB)) { - fail('database already exists ('.CONST_Database_DSN.')'); - } - - $sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database']; - if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username']; - } - - if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sCreateDBCmd .= ' -h ' . $this->aDSNInfo['hostspec']; - } - - $aProcEnv = null; - if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - - $result = runWithEnv($sCreateDBCmd, $aProcEnv); - if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); - } - - public function setupDB() - { - info('Setup DB'); - $this->oDB =& getDB(); - - $fPostgresVersion = getPostgresVersion($this->oDB); - echo 'Postgres version found: '.$fPostgresVersion."\n"; - - if ($fPostgresVersion < 9.1) { - fail('Minimum supported version of Postgresql is 9.1.'); - } - - $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore'); - $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); - - // For extratags and namedetails the hstore_to_json converter is - // needed which is only available from Postgresql 9.3+. For older - // versions add a dummy function that returns nothing. - $iNumFunc = chksql($this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); - - if ($iNumFunc == 0) { - $this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); - warn('Postgresql is too old. extratags and namedetails API not available.'); - } - - - $fPostgisVersion = getPostgisVersion($this->oDB); - echo 'Postgis version found: '.$fPostgisVersion."\n"; - - if ($fPostgisVersion < 2.1) { - // Functions were renamed in 2.1 and throw an annoying deprecation warning - $this->pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint'); - $this->pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint'); - } - if ($fPostgisVersion < 2.2) { - $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); - } - - $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'")); - if ($i == 0) { - echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; - echo "\n createuser ".CONST_Database_Web_User."\n\n"; - exit(1); - } - - if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { - echo 'Error: you need to download the country_osm_grid first:'; - echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; - exit(1); - } - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz'); - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql'); - - - if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz'); - } else { - warn('external UK postcode table not found.'); - } - - if (CONST_Use_Extra_US_Postcodes) { - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); - } - - if ($this->bNoPartitions) { - $this->pgsqlRunScript('update country_name set partition = 0'); - } - - // the following will be needed by create_functions later but - // is only defined in the subsequently called T - // Create dummies here that will be overwritten by the proper - // versions in create-tables. - $this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); - $this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); - } - - public function importData($sOSMFile) - { - info('Import data'); - - $osm2pgsql = CONST_Osm2pgsql_Binary; - if (!file_exists($osm2pgsql)) { - echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; - echo "Normally you should not need to set this manually.\n"; - fail("osm2pgsql not found in '$osm2pgsql'"); - } - - - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; - } - - if (CONST_Tablespace_Osm2pgsql_Data) - $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data; - if (CONST_Tablespace_Osm2pgsql_Index) - $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index; - if (CONST_Tablespace_Place_Data) - $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data; - if (CONST_Tablespace_Place_Index) - $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; - $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; - $osm2pgsql .= ' -C '.$this->iCacheMemory; - $osm2pgsql .= ' -P '.$this->aDSNInfo['port']; - if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $osm2pgsql .= ' -U ' . $this->aDSNInfo['username']; - } - if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $osm2pgsql .= ' -H ' . $this->aDSNInfo['hostspec']; - } - $aProcEnv = null; - if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - $osm2pgsql .= ' -d '.$this->aDSNInfo['database'].' '.$sOSMFile; - runWithEnv($osm2pgsql, $aProcEnv); - if ($this->oDB == null) $this->oDB =& getDB(); - if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) { - fail('No Data'); - } - } - - public function createFunctions() - { - info('Create Functions'); - - $this->createSqlFunctions(); - } - - public function createTables() - { - info('Create Tables'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = $this->replaceTablespace( - '{ts:address-data}', - CONST_Tablespace_Address_Data, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:search-data}', - CONST_Tablespace_Search_Data, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - - $this->pgsqlRunScript($sTemplate, false); - } - - public function recreateFunction() - { - // re-run the functions - info('Recreate Functions'); - $this->createSqlFunctions(); - } - - public function createPartitionTables() - { - info('Create Partition Tables'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); - $sTemplate = $this->replaceTablespace( - '{ts:address-data}', - CONST_Tablespace_Address_Data, - $sTemplate - ); - - $sTemplate = $this->replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - - $sTemplate = $this->replaceTablespace( - '{ts:search-data}', - CONST_Tablespace_Search_Data, - $sTemplate - ); - - $sTemplate = $this->replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - - $sTemplate = $this->replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - - $sTemplate = $this->replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - - $this->pgsqlRunPartitionScript($sTemplate); - } - - public function createPartitionFunctions() - { - info('Create Partition Functions'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); - $this->pgsqlRunPartitionScript($sTemplate); - } - - public function importWikipediaArticles() - { - $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; - $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; - if (file_exists($sWikiArticlesFile)) { - info('Importing wikipedia articles'); - $this->pgsqlRunDropAndRestore($sWikiArticlesFile); - } else { - warn('wikipedia article dump file not found - places will have default importance'); - } - if (file_exists($sWikiRedirectsFile)) { - info('Importing wikipedia redirects'); - $this->pgsqlRunDropAndRestore($sWikiRedirectsFile); - } else { - warn('wikipedia redirect dump file not found - some place importance values may be missing'); - } - echo ' finish wikipedia'; - } - - public function loadData($bDisableTokenPrecalc) - { - info('Drop old Data'); - - if ($this->oDB == null) $this->oDB =& getDB(); - - if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection)); - echo '.'; - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = chksql($this->oDB->getCol($sSQL)); - if (!$this->bNoPartitions) $aPartitions[] = 0; - foreach ($aPartitions as $sPartition) { - if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection)); - echo '.'; - } - - // used by getorcreate_word_id to ignore frequent partial words - $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; - $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - echo ".\n"; - - // pre-create the word list - if (!$bDisableTokenPrecalc) { - info('Loading word list'); - $this->pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql'); - } - - info('Load Data'); - $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; - $aDBInstances = array(); - $iLoadThreads = max(1, $this->iInstances - 1); - for ($i = 0; $i < $iLoadThreads; $i++) { - $aDBInstances[$i] =& getDB(true); - $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; - $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; - $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; - $sSQL .= ' and ST_IsValid(geometry)'; - if ($this->sVerbose) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { - fail(pg_last_error($aDBInstances[$i]->connection)); - } - } - - // last thread for interpolation lines - $aDBInstances[$iLoadThreads] =& getDB(true); - $sSQL = 'insert into location_property_osmline'; - $sSQL .= ' (osm_id, address, linegeo)'; - $sSQL .= ' SELECT osm_id, address, geometry from place where '; - $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; - if ($this->sVerbose) echo "$sSQL\n"; - if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { - fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); - } - - $bFailed = false; - for ($i = 0; $i <= $iLoadThreads; $i++) { - while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { - $resultStatus = pg_result_status($hPGresult); - // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, - // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, - // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR - echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; - if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { - $resultError = pg_result_error($hPGresult); - echo '-- error text ' . $i . ': ' . $resultError . "\n"; - $bFailed = true; - } - } - } - if ($bFailed) { - fail('SQL errors loading placex and/or location_property_osmline tables'); - } - echo "\n"; - info('Reanalysing database'); - $this->pgsqlRunScript('ANALYSE'); - - $sDatabaseDate = getDatabaseDate($this->oDB); - pg_query($this->oDB->connection, 'TRUNCATE import_status'); - if ($sDatabaseDate === false) { - warn('could not determine database date.'); - } else { - $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; - pg_query($this->oDB->connection, $sSQL); - echo "Latest data imported from $sDatabaseDate.\n"; - } - } - - public function importTigerData() - { - info('Import Tiger data'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = $this->replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - $this->pgsqlRunScript($sTemplate, false); - - $aDBInstances = array(); - for ($i = 0; $i < $this->iInstances; $i++) { - $aDBInstances[$i] =& getDB(true); - } - - foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) { - echo $sFile.': '; - $hFile = fopen($sFile, 'r'); - $sSQL = fgets($hFile, 100000); - $iLines = 0; - while (true) { - for ($i = 0; $i < $this->iInstances; $i++) { - if (!pg_connection_busy($aDBInstances[$i]->connection)) { - while (pg_get_result($aDBInstances[$i]->connection)); - $sSQL = fgets($hFile, 100000); - if (!$sSQL) break 2; - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); - $iLines++; - if ($iLines == 1000) { - echo '.'; - $iLines = 0; - } - } - } - usleep(10); - } - fclose($hFile); - - $bAnyBusy = true; - while ($bAnyBusy) { - $bAnyBusy = false; - for ($i = 0; $i < $this->iInstances; $i++) { - if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; - } - usleep(10); - } - echo "\n"; - } - - info('Creating indexes on Tiger data'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = $this->replaceTablespace( - '{ts:aux-data}', - CONST_Tablespace_Aux_Data, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - $this->pgsqlRunScript($sTemplate, false); - } - - public function calculatePostcodes($bCMDResultAll) - { - info('Calculate Postcodes'); - if ($this->oDB == null) $this->oDB =& getDB(); - if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { - fail(pg_last_error($this->oDB->connection)); - } - - - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; - $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; - $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; - $sSQL .= ' FROM placex'; - $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; - $sSQL .= ' AND geometry IS NOT null'; - $sSQL .= ' GROUP BY country_code, pc'; - - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - - if (CONST_Use_Extra_US_Postcodes) { - // only add postcodes that are not yet available in OSM - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; - $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; - $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'us')"; - if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); - } - - // add missing postcodes for GB (if available) - $sSQL = 'INSERT INTO location_postcode'; - $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; - $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; - $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; - $sSQL .= ' (SELECT postcode FROM location_postcode'; - $sSQL .= " WHERE country_code = 'gb')"; - if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); - - if (!$bCMDResultAll) { - $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; - $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - } - $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; - $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; - - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - } - - public function index($bIndexNoanalyse) - { - $sOutputFile = ''; - $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P ' - .$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile; - if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; - } - if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sBaseCmd .= ' -U ' . $this->aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - - info('Index ranks 0 - 4'); - $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); - info('Index ranks 5 - 25'); - $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); - info('Index ranks 26 - 30'); - $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); - if ($iStatus != 0) { - fail('error status ' . $iStatus . ' running nominatim!'); - } - info('Index postcodes'); - if ($this->oDB == null) $this->oDB =& getDB(); - $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; - if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); - } - - public function createSearchIndices() - { - info('Create Search indices'); - - $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); - $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); - $sTemplate = $this->replaceTablespace( - '{ts:address-index}', - CONST_Tablespace_Address_Index, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:search-index}', - CONST_Tablespace_Search_Index, - $sTemplate - ); - $sTemplate = $this->replaceTablespace( - '{ts:aux-index}', - CONST_Tablespace_Aux_Index, - $sTemplate - ); - $this->pgsqlRunScript($sTemplate); - } - - public function createCountryNames() - { - info('Create search index for default country names'); - - $this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); - $this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); - $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); - $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); - $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),' - .'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; - if (CONST_Languages) { - $sSQL .= 'in '; - $sDelim = '('; - foreach (explode(',', CONST_Languages) as $sLang) { - $sSQL .= $sDelim."'name:$sLang'"; - $sDelim = ','; - } - $sSQL .= ')'; - } else { - // all include all simple name tags - $sSQL .= "like 'name:%'"; - } - $sSQL .= ') v'; - $this->pgsqlRunScript($sSQL); - } - - public function drop() - { - info('Drop tables only required for updates'); - - // The implementation is potentially a bit dangerous because it uses - // a positive selection of tables to keep, and deletes everything else. - // Including any tables that the unsuspecting user might have manually - // created. USE AT YOUR OWN PERIL. - // tables we want to keep. everything else goes. - $aKeepTables = array( - '*columns', - 'import_polygon_*', - 'import_status', - 'place_addressline', - 'location_postcode', - 'location_property*', - 'placex', - 'search_name', - 'seq_*', - 'word', - 'query_log', - 'new_query_log', - 'spatial_ref_sys', - 'country_name', - 'place_classtype_*' - ); - - if ($this->oDB = null) $this->oDB =& getDB(); - $aDropTables = array(); - $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); - - foreach ($aHaveTables as $sTable) { - $bFound = false; - foreach ($aKeepTables as $sKeep) { - if (fnmatch($sKeep, $sTable)) { - $bFound = true; - break; - } - } - if (!$bFound) array_push($aDropTables, $sTable); - } - foreach ($aDropTables as $sDrop) { - if ($this->sVerbose) echo "dropping table $sDrop\n"; - @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE"); - // ignore warnings/errors as they might be caused by a table having - // been deleted already by CASCADE - } - - if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; - unlink(CONST_Osm2pgsql_Flatnode_File); - } - } - - private function pgsqlRunDropAndRestore($sDumpFile) - { - if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; - $sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile; - if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; - } - if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sCMD .= ' -U ' . $this->aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" - } - - private function pgsqlRunScript($sScript, $bfatal = true) - { - runSQLScript( - $sScript, - $bfatal, - $this->sVerbose, - $this->sIgnoreErrors - ); - } - - private function createSqlFunctions() - { - $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); - $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); - if ($this->bEnableDiffUpdates) { - $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); - } - if ($this->bEnableDebugStatements) { - $sTemplate = str_replace('--DEBUG:', '', $sTemplate); - } - if (CONST_Limit_Reindexing) { - $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); - } - if (!CONST_Use_US_Tiger_Data) { - $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); - } - if (!CONST_Use_Aux_Location_data) { - $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); - } - $this->pgsqlRunScript($sTemplate); - } - - private function pgsqlRunPartitionScript($sTemplate) - { - if ($this->oDB == null) $this->oDB =& getDB(); - - $sSQL = 'select distinct partition from country_name'; - $aPartitions = chksql($this->oDB->getCol($sSQL)); - if (!$this->bNoPartitions) $aPartitions[] = 0; - - preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); - foreach ($aMatches as $aMatch) { - $sResult = ''; - foreach ($aPartitions as $sPartitionName) { - $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); - } - $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); - } - - $this->pgsqlRunScript($sTemplate); - } - - private function pgsqlRunScriptFile($sFilename) - { - if (!file_exists($sFilename)) fail('unable to find '.$sFilename); - - $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database']; - if (!$this->sVerbose) { - $sCMD .= ' -q'; - } - if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; - } - if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sCMD .= ' -U ' . $this->aDSNInfo['username']; - } - $aProcEnv = null; - if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { - $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); - } - $ahGzipPipes = null; - if (preg_match('/\\.gz$/', $sFilename)) { - $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); - if (!is_resource($hGzipProcess)) fail('unable to start zcat'); - $aReadPipe = $ahGzipPipes[1]; - fclose($ahGzipPipes[0]); - } else { - $sCMD .= ' -f '.$sFilename; - $aReadPipe = array('pipe', 'r'); - } - $aDescriptors = array( - 0 => $aReadPipe, - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a') - ); - $ahPipes = null; - $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); - if (!is_resource($hProcess)) fail('unable to start pgsql'); - // TODO: error checking - while (!feof($ahPipes[1])) { - echo fread($ahPipes[1], 4096); - } - fclose($ahPipes[1]); - $iReturn = proc_close($hProcess); - if ($iReturn > 0) { - fail("pgsql returned with error code ($iReturn)"); - } - if ($ahGzipPipes) { - fclose($ahGzipPipes[1]); - proc_close($hGzipProcess); - } - } - - private function replaceTablespace($sTemplate, $sTablespace, $sSql) - { - if ($sTablespace) { - $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); - } else { - $sSql = str_replace($sTemplate, '', $sSql); - } - return $sSql; - } -} +iInstances = isset($aCMDResult['threads']) + ? $aCMDResult['threads'] + : (min(16, getProcessorCount()) - 1); + + if ($this->iInstances < 1) { + $this->iInstances = 1; + warn('resetting threads to ' . $this->iInstances); + } + + // Assume we can steal all the cache memory in the box (unless told otherwise) + if (isset($aCMDResult['osm2pgsql-cache'])) { + $this->iCacheMemory = $aCMDResult['osm2pgsql-cache']; + } else { + $this->iCacheMemory = getCacheMemoryMB(); + } + + $this->sModulePath = CONST_Database_Module_Path; + info('module path: ' . $this->sModulePath); + + // prepares DB for import or update, sets the Data Source Name + $this->aDSNInfo = \DB::parseDSN(CONST_Database_DSN); + if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) { + $this->aDSNInfo['port'] = 5432; + } + + // setting member variables based on command line options stored in $aCMDResult + $this->sVerbose = $aCMDResult['verbose']; + $this->sIgnoreErrors = $aCMDResult['ignore-errors']; + $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; + $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; + $this->bNoPartitions = $aCMDResult['no-partitions']; + } + + public function createDB() + { + info('Create DB'); + $sDB = \DB::connect(CONST_Database_DSN, false); + if (!\PEAR::isError($sDB)) { + fail('database already exists (' . CONST_Database_DSN . ')'); + } + + $sCreateDBCmd = 'createdb -E UTF-8 -p ' . $this->aDSNInfo['port'] . ' ' . $this->aDSNInfo['database']; + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username']; + } + + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCreateDBCmd .= ' -h ' . $this->aDSNInfo['hostspec']; + } + + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + + $result = runWithEnv($sCreateDBCmd, $aProcEnv); + if ($result != 0) { + fail('Error executing external command: ' . $sCreateDBCmd); + } + + } + + public function setupDB() + { + info('Setup DB'); + $this->oDB = &getDB(); + + $fPostgresVersion = getPostgresVersion($this->oDB); + echo 'Postgres version found: ' . $fPostgresVersion . "\n"; + + if ($fPostgresVersion < 9.1) { + fail('Minimum supported version of Postgresql is 9.1.'); + } + + $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore'); + $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis'); + + // For extratags and namedetails the hstore_to_json converter is + // needed which is only available from Postgresql 9.3+. For older + // versions add a dummy function that returns nothing. + $iNumFunc = chksql($this->oDB->getOne("select count(*) from pg_proc where proname = 'hstore_to_json'")); + + if ($iNumFunc == 0) { + $this->pgsqlRunScript("create function hstore_to_json(dummy hstore) returns text AS 'select null::text' language sql immutable"); + warn('Postgresql is too old. extratags and namedetails API not available.'); + } + + $fPostgisVersion = getPostgisVersion($this->oDB); + echo 'Postgis version found: ' . $fPostgisVersion . "\n"; + + if ($fPostgisVersion < 2.1) { + // Functions were renamed in 2.1 and throw an annoying deprecation warning + $this->pgsqlRunScript('ALTER FUNCTION st_line_interpolate_point(geometry, double precision) RENAME TO ST_LineInterpolatePoint'); + $this->pgsqlRunScript('ALTER FUNCTION ST_Line_Locate_Point(geometry, geometry) RENAME TO ST_LineLocatePoint'); + } + if ($fPostgisVersion < 2.2) { + $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); + } + + $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '" . CONST_Database_Web_User . "'")); + if ($i == 0) { + echo "\nERROR: Web user '" . CONST_Database_Web_User . "' does not exist. Create it with:\n"; + echo "\n createuser " . CONST_Database_Web_User . "\n\n"; + exit(1); + } + + if (!file_exists(CONST_ExtraDataPath . '/country_osm_grid.sql.gz')) { + echo 'Error: you need to download the country_osm_grid first:'; + echo "\n wget -O " . CONST_ExtraDataPath . "/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; + exit(1); + } + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_name.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_naturalearthdata.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_osm_grid.sql.gz'); + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_table.sql'); + + if (file_exists(CONST_BasePath . '/data/gb_postcode_data.sql.gz')) { + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_data.sql.gz'); + } else { + warn('external UK postcode table not found.'); + } + + if (CONST_Use_Extra_US_Postcodes) { + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/us_postcode.sql'); + } + + if ($this->bNoPartitions) { + $this->pgsqlRunScript('update country_name set partition = 0'); + } + + // the following will be needed by create_functions later but + // is only defined in the subsequently called T + // Create dummies here that will be overwritten by the proper + // versions in create-tables. + $this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); + $this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); + } + + public function importData($sOSMFile) + { + info('Import data'); + + $osm2pgsql = CONST_Osm2pgsql_Binary; + if (!file_exists($osm2pgsql)) { + echo "Check CONST_Osm2pgsql_Binary in your local settings file.\n"; + echo "Normally you should not need to set this manually.\n"; + fail("osm2pgsql not found in '$osm2pgsql'"); + } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { + $osm2pgsql .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File; + } + + if (CONST_Tablespace_Osm2pgsql_Data) { + $osm2pgsql .= ' --tablespace-slim-data ' . CONST_Tablespace_Osm2pgsql_Data; + } + + if (CONST_Tablespace_Osm2pgsql_Index) { + $osm2pgsql .= ' --tablespace-slim-index ' . CONST_Tablespace_Osm2pgsql_Index; + } + + if (CONST_Tablespace_Place_Data) { + $osm2pgsql .= ' --tablespace-main-data ' . CONST_Tablespace_Place_Data; + } + + if (CONST_Tablespace_Place_Index) { + $osm2pgsql .= ' --tablespace-main-index ' . CONST_Tablespace_Place_Index; + } + + $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; + $osm2pgsql .= ' -C ' . $this->iCacheMemory; + $osm2pgsql .= ' -P ' . $this->aDSNInfo['port']; + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $osm2pgsql .= ' -U ' . $this->aDSNInfo['username']; + } + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $osm2pgsql .= ' -H ' . $this->aDSNInfo['hostspec']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $osm2pgsql .= ' -d ' . $this->aDSNInfo['database'] . ' ' . $sOSMFile; + runWithEnv($osm2pgsql, $aProcEnv); + if ($this->oDB == null) { + $this->oDB = &getDB(); + } + + if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) { + fail('No Data'); + } + } + + public function createFunctions() + { + info('Create Functions'); + + $this->createSqlFunctions(); + } + + public function createTables() + { + info('Create Tables'); + + $sTemplate = file_get_contents(CONST_BasePath . '/sql/tables.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:address-data}', + CONST_Tablespace_Address_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-data}', + CONST_Tablespace_Search_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + + $this->pgsqlRunScript($sTemplate, false); + } + + public function createPartitionTables() + { + info('Create Partition Tables'); + + $sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-tables.src.sql'); + $sTemplate = $this->replaceTablespace( + '{ts:address-data}', + CONST_Tablespace_Address_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:search-data}', + CONST_Tablespace_Search_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + + $this->pgsqlRunPartitionScript($sTemplate); + } + + public function createPartitionFunctions() + { + info('Create Partition Functions'); + + $sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-functions.src.sql'); + $this->pgsqlRunPartitionScript($sTemplate); + } + + public function importWikipediaArticles() + { + $sWikiArticlesFile = CONST_Wikipedia_Data_Path . '/wikipedia_article.sql.bin'; + $sWikiRedirectsFile = CONST_Wikipedia_Data_Path . '/wikipedia_redirect.sql.bin'; + if (file_exists($sWikiArticlesFile)) { + info('Importing wikipedia articles'); + $this->pgsqlRunDropAndRestore($sWikiArticlesFile); + } else { + warn('wikipedia article dump file not found - places will have default importance'); + } + if (file_exists($sWikiRedirectsFile)) { + info('Importing wikipedia redirects'); + $this->pgsqlRunDropAndRestore($sWikiRedirectsFile); + } else { + warn('wikipedia redirect dump file not found - some place importance values may be missing'); + } + echo ' finish wikipedia'; + } + + public function loadData($bDisableTokenPrecalc) + { + info('Drop old Data'); + + if ($this->oDB == null) { + $this->oDB = &getDB(); + } + + if (!pg_query($this->oDB->connection, 'TRUNCATE word')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($this->oDB->getCol($sSQL)); + if (!$this->bNoPartitions) { + $aPartitions[] = 0; + } + + foreach ($aPartitions as $sPartition) { + if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_' . $sPartition)) { + fail(pg_last_error($this->oDB->connection)); + } + + echo '.'; + } + + // used by getorcreate_word_id to ignore frequent partial words + $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; + $sSQL .= '$$ SELECT ' . CONST_Max_Word_Frequency . ' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + echo ".\n"; + + // pre-create the word list + if (!$bDisableTokenPrecalc) { + info('Loading word list'); + $this->pgsqlRunScriptFile(CONST_BasePath . '/data/words.sql'); + } + + info('Load Data'); + $sColumns = 'osm_type, osm_id, class, type, name, admin_level, address, extratags, geometry'; + $aDBInstances = array(); + $iLoadThreads = max(1, $this->iInstances - 1); + for ($i = 0; $i < $iLoadThreads; $i++) { + $aDBInstances[$i] = &getDB(true); + $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; + $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; + $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; + $sSQL .= ' and ST_IsValid(geometry)'; + if ($this->sVerbose) { + echo "$sSQL\n"; + } + + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { + fail(pg_last_error($aDBInstances[$i]->connection)); + } + } + + // last thread for interpolation lines + $aDBInstances[$iLoadThreads] = &getDB(true); + $sSQL = 'insert into location_property_osmline'; + $sSQL .= ' (osm_id, address, linegeo)'; + $sSQL .= ' SELECT osm_id, address, geometry from place where '; + $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; + if ($this->sVerbose) { + echo "$sSQL\n"; + } + + if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { + fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); + } + + $bFailed = false; + for ($i = 0; $i <= $iLoadThreads; $i++) { + while (($hPGresult = pg_get_result($aDBInstances[$i]->connection)) !== false) { + $resultStatus = pg_result_status($hPGresult); + // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, + // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, + // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR + echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; + if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { + $resultError = pg_result_error($hPGresult); + echo '-- error text ' . $i . ': ' . $resultError . "\n"; + $bFailed = true; + } + } + } + if ($bFailed) { + fail('SQL errors loading placex and/or location_property_osmline tables'); + } + echo "\n"; + info('Reanalysing database'); + $this->pgsqlRunScript('ANALYSE'); + + $sDatabaseDate = getDatabaseDate($this->oDB); + pg_query($this->oDB->connection, 'TRUNCATE import_status'); + if ($sDatabaseDate === false) { + warn('could not determine database date.'); + } else { + $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('" . $sDatabaseDate . "')"; + pg_query($this->oDB->connection, $sSQL); + echo "Latest data imported from $sDatabaseDate.\n"; + } + } + + public function importTigerData() + { + info('Import Tiger data'); + + $sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_start.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate, false); + + $aDBInstances = array(); + for ($i = 0; $i < $this->iInstances; $i++) { + $aDBInstances[$i] = &getDB(true); + } + + foreach (glob(CONST_Tiger_Data_Path . '/*.sql') as $sFile) { + echo $sFile . ': '; + $hFile = fopen($sFile, 'r'); + $sSQL = fgets($hFile, 100000); + $iLines = 0; + while (true) { + for ($i = 0; $i < $this->iInstances; $i++) { + if (!pg_connection_busy($aDBInstances[$i]->connection)) { + while (pg_get_result($aDBInstances[$i]->connection)); + $sSQL = fgets($hFile, 100000); + if (!$sSQL) { + break 2; + } + + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + $iLines++; + if ($iLines == 1000) { + echo '.'; + $iLines = 0; + } + } + } + usleep(10); + } + fclose($hFile); + + $bAnyBusy = true; + while ($bAnyBusy) { + $bAnyBusy = false; + for ($i = 0; $i < $this->iInstances; $i++) { + if (pg_connection_busy($aDBInstances[$i]->connection)) { + $bAnyBusy = true; + } + + } + usleep(10); + } + echo "\n"; + } + + info('Creating indexes on Tiger data'); + $sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_finish.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:aux-data}', + CONST_Tablespace_Aux_Data, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate, false); + } + + public function calculatePostcodes($bCMDResultAll) + { + info('Calculate Postcodes'); + if ($this->oDB == null) { + $this->oDB = &getDB(); + } + + if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { + fail(pg_last_error($this->oDB->connection)); + } + + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; + $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; + $sSQL .= ' ST_Centroid(ST_Collect(ST_Centroid(geometry)))'; + $sSQL .= ' FROM placex'; + $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; + $sSQL .= ' AND geometry IS NOT null'; + $sSQL .= ' GROUP BY country_code, pc'; + + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + if (CONST_Use_Extra_US_Postcodes) { + // only add postcodes that are not yet available in OSM + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; + $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; + $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; + $sSQL .= ' (SELECT postcode FROM location_postcode'; + $sSQL .= " WHERE country_code = 'us')"; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + } + + // add missing postcodes for GB (if available) + $sSQL = 'INSERT INTO location_postcode'; + $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; + $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; + $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; + $sSQL .= ' (SELECT postcode FROM location_postcode'; + $sSQL .= " WHERE country_code = 'gb')"; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + if (!$bCMDResultAll) { + $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; + $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + } + $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; + $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; + + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + } + + public function index($bIndexNoanalyse) + { + $sOutputFile = ''; + $sBaseCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $this->aDSNInfo['database'] . ' -P ' + . $this->aDSNInfo['port'] . ' -t ' . $this->iInstances . $sOutputFile; + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sBaseCmd .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + + info('Index ranks 0 - 4'); + $iStatus = runWithEnv($sBaseCmd . ' -R 4', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + if (!$bIndexNoanalyse) { + $this->pgsqlRunScript('ANALYSE'); + } + + info('Index ranks 5 - 25'); + $iStatus = runWithEnv($sBaseCmd . ' -r 5 -R 25', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + if (!$bIndexNoanalyse) { + $this->pgsqlRunScript('ANALYSE'); + } + + info('Index ranks 26 - 30'); + $iStatus = runWithEnv($sBaseCmd . ' -r 26', $aProcEnv); + if ($iStatus != 0) { + fail('error status ' . $iStatus . ' running nominatim!'); + } + info('Index postcodes'); + if ($this->oDB == null) { + $this->oDB = &getDB(); + } + + $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; + if (!pg_query($this->oDB->connection, $sSQL)) { + fail(pg_last_error($this->oDB->connection)); + } + + } + + public function createSearchIndices() + { + info('Create Search indices'); + + $sTemplate = file_get_contents(CONST_BasePath . '/sql/indices.src.sql'); + $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); + $sTemplate = $this->replaceTablespace( + '{ts:address-index}', + CONST_Tablespace_Address_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:search-index}', + CONST_Tablespace_Search_Index, + $sTemplate + ); + $sTemplate = $this->replaceTablespace( + '{ts:aux-index}', + CONST_Tablespace_Aux_Index, + $sTemplate + ); + $this->pgsqlRunScript($sTemplate); + } + + public function createCountryNames() + { + info('Create search index for default country names'); + + $this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); + $this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); + $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); + $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); + $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),' + . 'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; + if (CONST_Languages) { + $sSQL .= 'in '; + $sDelim = '('; + foreach (explode(',', CONST_Languages) as $sLang) { + $sSQL .= $sDelim . "'name:$sLang'"; + $sDelim = ','; + } + $sSQL .= ')'; + } else { + // all include all simple name tags + $sSQL .= "like 'name:%'"; + } + $sSQL .= ') v'; + $this->pgsqlRunScript($sSQL); + } + + public function drop() + { + info('Drop tables only required for updates'); + + // The implementation is potentially a bit dangerous because it uses + // a positive selection of tables to keep, and deletes everything else. + // Including any tables that the unsuspecting user might have manually + // created. USE AT YOUR OWN PERIL. + // tables we want to keep. everything else goes. + $aKeepTables = array( + '*columns', + 'import_polygon_*', + 'import_status', + 'place_addressline', + 'location_postcode', + 'location_property*', + 'placex', + 'search_name', + 'seq_*', + 'word', + 'query_log', + 'new_query_log', + 'spatial_ref_sys', + 'country_name', + 'place_classtype_*', + ); + + if ($this->oDB = null) { + $this->oDB = &getDB(); + } + + $aDropTables = array(); + $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); + + foreach ($aHaveTables as $sTable) { + $bFound = false; + foreach ($aKeepTables as $sKeep) { + if (fnmatch($sKeep, $sTable)) { + $bFound = true; + break; + } + } + if (!$bFound) { + array_push($aDropTables, $sTable); + } + + } + foreach ($aDropTables as $sDrop) { + if ($this->sVerbose) { + echo "dropping table $sDrop\n"; + } + + @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE"); + // ignore warnings/errors as they might be caused by a table having + // been deleted already by CASCADE + } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { + if ($sVerbose) { + echo 'deleting ' . CONST_Osm2pgsql_Flatnode_File . "\n"; + } + + unlink(CONST_Osm2pgsql_Flatnode_File); + } + } + + private function pgsqlRunDropAndRestore($sDumpFile) + { + if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) { + $this->aDSNInfo['port'] = 5432; + } + + $sCMD = 'pg_restore -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database'] . ' -Fc --clean ' . $sDumpFile; + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCMD .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" + } + + private function pgsqlRunScript($sScript, $bfatal = true) + { + runSQLScript( + $sScript, + $bfatal, + $this->sVerbose, + $this->sIgnoreErrors + ); + } + + private function createSqlFunctions() + { + $sTemplate = file_get_contents(CONST_BasePath . '/sql/functions.sql'); + $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); + if ($this->bEnableDiffUpdates) { + $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); + } + if ($this->bEnableDebugStatements) { + $sTemplate = str_replace('--DEBUG:', '', $sTemplate); + } + if (CONST_Limit_Reindexing) { + $sTemplate = str_replace('--LIMIT INDEXING:', '', $sTemplate); + } + if (!CONST_Use_US_Tiger_Data) { + $sTemplate = str_replace('-- %NOTIGERDATA% ', '', $sTemplate); + } + if (!CONST_Use_Aux_Location_data) { + $sTemplate = str_replace('-- %NOAUXDATA% ', '', $sTemplate); + } + $this->pgsqlRunScript($sTemplate); + } + + private function pgsqlRunPartitionScript($sTemplate) + { + if ($this->oDB == null) { + $this->oDB = &getDB(); + } + + $sSQL = 'select distinct partition from country_name'; + $aPartitions = chksql($this->oDB->getCol($sSQL)); + if (!$this->bNoPartitions) { + $aPartitions[] = 0; + } + + preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); + foreach ($aMatches as $aMatch) { + $sResult = ''; + foreach ($aPartitions as $sPartitionName) { + $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); + } + $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); + } + + $this->pgsqlRunScript($sTemplate); + } + + private function pgsqlRunScriptFile($sFilename) + { + if (!file_exists($sFilename)) { + fail('unable to find ' . $sFilename); + } + + $sCMD = 'psql -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database']; + if (!$this->sVerbose) { + $sCMD .= ' -q'; + } + if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { + $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; + } + if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { + $sCMD .= ' -U ' . $this->aDSNInfo['username']; + } + $aProcEnv = null; + if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { + $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); + } + $ahGzipPipes = null; + if (preg_match('/\\.gz$/', $sFilename)) { + $aDescriptors = array( + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a'), + ); + $hGzipProcess = proc_open('zcat ' . $sFilename, $aDescriptors, $ahGzipPipes); + if (!is_resource($hGzipProcess)) { + fail('unable to start zcat'); + } + + $aReadPipe = $ahGzipPipes[1]; + fclose($ahGzipPipes[0]); + } else { + $sCMD .= ' -f ' . $sFilename; + $aReadPipe = array('pipe', 'r'); + } + $aDescriptors = array( + 0 => $aReadPipe, + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a'), + ); + $ahPipes = null; + $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); + if (!is_resource($hProcess)) { + fail('unable to start pgsql'); + } + + // TODO: error checking + while (!feof($ahPipes[1])) { + echo fread($ahPipes[1], 4096); + } + fclose($ahPipes[1]); + $iReturn = proc_close($hProcess); + if ($iReturn > 0) { + fail("pgsql returned with error code ($iReturn)"); + } + if ($ahGzipPipes) { + fclose($ahGzipPipes[1]); + proc_close($hGzipProcess); + } + } + + private function replaceTablespace($sTemplate, $sTablespace, $sSql) + { + if ($sTablespace) { + $sSql = str_replace($sTemplate, 'TABLESPACE "' . $sTablespace . '"', $sSql); + } else { + $sSql = str_replace($sTemplate, '', $sSql); + } + return $sSql; + } +} diff --git a/lib/setup_functions.php b/lib/setup_functions.php index fec28f17..7b705168 100755 --- a/lib/setup_functions.php +++ b/lib/setup_functions.php @@ -1,80 +1,126 @@ -query($sSQL); - - $bResult = true; - - if (PEAR::isError($oResult)) { - echo "\nERROR: Failed to load nominatim module. Reason:\n"; - echo $oResult->userinfo."\n\n"; - $bResult = false; - } - - return $bResult; -} - -function createSetupArgvArray() -{ - $aCMDOptions - = array( - 'Create and setup nominatim search system', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - - array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), - array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), - - array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), - - array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), - array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), - array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), - array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), - array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), - array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), - array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), - array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), - array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), - array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), - array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), - array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), - array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), - array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), - array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), - array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), - array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), - array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), - array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), - array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), - ); - return $aCMDOptions; -} +query($sSQL); + + $bResult = true; + + if (PEAR::isError($oResult)) { + echo "\nERROR: Failed to load nominatim module. Reason:\n"; + echo $oResult->userinfo . "\n\n"; + $bResult = false; + } + + return $bResult; +} + +// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) +// create and array +function createSetupArgvArray() +{ + $aCMDOptions + = array( + 'Create and setup nominatim search system', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), + array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), + + array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), + + array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), + array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), + array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), + array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), + array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), + array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), + array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), + array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), + array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), + array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), + array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), + array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), + array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), + array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), + array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), + array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), + array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), + array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), + array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), + array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), + ); + return $aCMDOptions; +} + +function createUpdateArgvArray() +{ + $aCMDOptions + = array( + 'Import / update / index osm data', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), + array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), + array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'), + array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), + array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), + array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), + + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'), + + array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'), + array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + + array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'), + array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'), + array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'), + array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'), + + array('index', '', 0, 1, 0, 0, 'bool', 'Index'), + array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'), + array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'), + + array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'), + array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'), + array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'), + + array('create-functions', '', 0, 1, 1, 1, 'bool', 'Create functions'), + array('enable-diff-updates', '', 0, 1, 1, 1, 'bool', 'Turn on the code required to make diff updates work'), + array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), + array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), + array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), + ); + return $aCMDOptions; +} diff --git a/utils/setup.php b/utils/setup.php index 8eb90cfa..989d28e1 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -1,133 +1,133 @@ -#!@PHP_BIN@ -Cq -indirect via init-cmd.php -> /lib/cmd.php for runWithEnv, getCmdOpt -// ->indirect via init-cmd.php -> /lib/init.php -> db.php for &getDB() - -include_once(CONST_BasePath.'/lib/setup_functions.php'); -include_once(CONST_BasePath.'/lib/setup_functions.php'); -ini_set('memory_limit', '800M'); - - -$aCMDOptions = createSetupArgvArray(); - -// $aCMDOptions passed to getCmdOpt by reference -getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); - - -$bDidSomething = false; - -//******************************************************* -// Making some sanity check: -// Check if osm-file is set and points to a valid file -if ($aCMDResult['import-data'] || $aCMDResult['all']) { - // to remain in /lib/setup_functions.php function - checkInFile($aCMDResult['osm-file']); - echo $aCMDResult['osm-file']; -} - -// osmosis init is no longer supported -if ($aCMDResult['osmosis-init']) { - $bDidSomething = true; - echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n"; -} - -// ****************************************************** -// instantiate Setup class -$cSetup = new SetupFunctions($aCMDResult); -if ($aCMDResult['create-db'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createDB(); -} - -// ******************************************************* -// go through complete process if 'all' is selected or start selected functions -if ($aCMDResult['setup-db'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> setupDB(); -} - -// Try accessing the C module, so we know early if something is wrong -if (!checkModulePresence()) { - fail('error loading nominatim.so module'); -} - -if ($aCMDResult['import-data'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> importData($aCMDResult['osm-file']); -} - -if ($aCMDResult['create-functions'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createFunctions(); -} - -if ($aCMDResult['create-tables'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createTables(); - $cSetup -> recreateFunction(); -} - -if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createPartitionTables(); -} - -if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createPartitionFunctions(); -} - -if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> importWikipediaArticles(); -} - -if ($aCMDResult['load-data'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> loadData($aCMDResult['disable-token-precalc']); -} - -if ($aCMDResult['import-tiger-data']) { - $bDidSomething = true; - $cSetup -> importTigerData(); -} - -if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> calculatePostcodes($aCMDResult['all']); -} - -if ($aCMDResult['index'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> index($aCMDResult['index-noanalyse']); -} - -if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createSearchIndices(); -} - -if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { - $bDidSomething = true; - $cSetup -> createCountryNames($aCMDResult); -} - -if ($aCMDResult['drop']) { - $bDidSomething = true; - $cSetup -> drop($aCMDResult); -} - -// ****************************************************** -// If we did something, repeat the warnings -if (!$bDidSomething) { - showUsage($aCMDOptions, true); -} else { - echo "Summary of warnings:\n\n"; - repeatWarnings(); - echo "\n"; - info('Setup finished.'); -} +#!@PHP_BIN@ -Cq +indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt +// ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB() + +require_once CONST_BasePath . '/lib/setup_functions.php'; +ini_set('memory_limit', '800M'); + +use Nominatim\Setup\SetupFunctions as SetupFunctions; + +$aCMDOptions = createSetupArgvArray(); + +// $aCMDOptions passed to getCmdOpt by reference +getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); + +$bDidSomething = false; + +//******************************************************* +// Making some sanity check: +// Check if osm-file is set and points to a valid file +if ($aCMDResult['import-data'] || $aCMDResult['all']) { + // to remain in /lib/setup_functions.php function + checkInFile($aCMDResult['osm-file']); + echo $aCMDResult['osm-file']; +} + +// osmosis init is no longer supported +if ($aCMDResult['osmosis-init']) { + $bDidSomething = true; + echo "Command 'osmosis-init' no longer available, please use utils/update.php --init-updates.\n"; +} + +// ****************************************************** +// instantiate Setup class +$cSetup = new SetupFunctions($aCMDResult); + +// ******************************************************* +// go through complete process if 'all' is selected or start selected functions +if ($aCMDResult['create-db'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createDB(); +} + +if ($aCMDResult['setup-db'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->setupDB(); +} + +// Try accessing the C module, so we know early if something is wrong +if (!checkModulePresence()) { + fail('error loading nominatim.so module'); +} + +if ($aCMDResult['import-data'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->importData($aCMDResult['osm-file']); +} + +if ($aCMDResult['create-functions'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createFunctions(); +} + +if ($aCMDResult['create-tables'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createTables(); + $cSetup->createFunctions(); +} + +if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createPartitionTables(); +} + +if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createPartitionFunctions(); +} + +if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->importWikipediaArticles(); +} + +if ($aCMDResult['load-data'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->loadData($aCMDResult['disable-token-precalc']); +} + +if ($aCMDResult['import-tiger-data']) { + $bDidSomething = true; + $cSetup->importTigerData(); +} + +if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->calculatePostcodes($aCMDResult['all']); +} + +if ($aCMDResult['index'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->index($aCMDResult['index-noanalyse']); +} + +if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createSearchIndices(); +} + +if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { + $bDidSomething = true; + $cSetup->createCountryNames($aCMDResult); +} + +if ($aCMDResult['drop']) { + $bDidSomething = true; + $cSetup->drop($aCMDResult); +} + +// ****************************************************** +// If we did something, repeat the warnings +if (!$bDidSomething) { + showUsage($aCMDOptions, true); +} else { + echo "Summary of warnings:\n\n"; + repeatWarnings(); + echo "\n"; + info('Setup finished.'); +} diff --git a/utils/update.php b/utils/update.php index fd361eb8..176bde06 100755 --- a/utils/update.php +++ b/utils/update.php @@ -1,66 +1,42 @@ #!@PHP_BIN@ -Cq getTotalMemoryMB()) { $iCacheMemory = getCacheMemoryMB(); echo "WARNING: resetting cache memory to $iCacheMemory\n"; } -$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; +$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary . ' -klas --number-processes 1 -C ' . $iCacheMemory . ' -O gazetteer -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port']; if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { $sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username']; } @@ -73,12 +49,12 @@ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { } if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; + $sOsm2pgsqlCmd .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File; } if ($aResult['init-updates']) { // sanity check that the replication URL is correct - $sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt'); + $sBaseState = file_get_contents(CONST_Replication_Url . '/state.txt'); if ($sBaseState === false) { echo "\nCannot find state.txt file at the configured replication URL.\n"; echo "Does the URL point to a directory containing OSM update data?\n\n"; @@ -92,7 +68,7 @@ if ($aResult['init-updates']) { fail('CONST_Pyosmium_Binary not configured'); } $aOutput = 0; - $sCmd = CONST_Pyosmium_Binary.' --help'; + $sCmd = CONST_Pyosmium_Binary . ' --help'; exec($sCmd, $aOutput, $iRet); if ($iRet != 0) { echo "Cannot execute pyosmium-get-changes.\n"; @@ -100,19 +76,23 @@ if ($aResult['init-updates']) { echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n"; fail('pyosmium-get-changes not found or not usable'); } + if (!$aResult['no-update-functions']) { - createFunctions($aCMDResult); + // instatiate setupClass to use the function therein + // instantiate Setup class + $cSetup = new SetupFunctions($aResult); + $cSetup->createFunctions(); } $sDatabaseDate = getDatabaseDate($oDB); if ($sDatabaseDate === false) { fail('Cannot determine date of database.'); } - $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60)); + $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3 * 60 * 60)); // get the appropriate state id $aOutput = 0; - $sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url; + $sCmd = CONST_Pyosmium_Binary . ' -D ' . $sWindBack . ' --server ' . CONST_Replication_Url; exec($sCmd, $aOutput, $iRet); if ($iRet != 0 || $aOutput[0] == 'None') { fail('Error running pyosmium tools'); @@ -120,7 +100,7 @@ if ($aResult['init-updates']) { pg_query($oDB->connection, 'TRUNCATE import_status'); $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('"; - $sSQL .= $sDatabaseDate."',".$aOutput[0].', true)'; + $sSQL .= $sDatabaseDate . "'," . $aOutput[0] . ', true)'; if (!pg_query($oDB->connection, $sSQL)) { fail('Could not enter sequence into database.'); } @@ -135,7 +115,7 @@ if ($aResult['check-for-updates']) { fail('Updates not set up. Please run ./utils/update.php --init-updates.'); } - system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet); + system(CONST_BasePath . '/utils/check_server_for_updates.py ' . CONST_Replication_Url . ' ' . $aLastState['sequence_id'], $iRet); exit($iRet); } @@ -148,8 +128,8 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { } // Import the file - $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile; - echo $sCMD."\n"; + $sCMD = $sOsm2pgsqlCmd . ' ' . $sNextFile; + echo $sCMD . "\n"; $iErrorLevel = runWithEnv($sCMD, $aProcEnv); if ($iErrorLevel) { @@ -161,35 +141,35 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { if ($aResult['calculate-postcodes']) { info('Update postcodes centroids'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql'); + $sTemplate = file_get_contents(CONST_BasePath . '/sql/update-postcodes.sql'); runSQLScript($sTemplate, true, true); } -$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc'; +$sTemporaryFile = CONST_BasePath . '/data/osmosischange.osc'; $bHaveDiff = false; $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api']; $sContentURL = ''; if (isset($aResult['import-node']) && $aResult['import-node']) { if ($bUseOSMApi) { - $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node']; + $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/' . $aResult['import-node']; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=node(' . $aResult['import-node'] . ');out%20meta;'; } } if (isset($aResult['import-way']) && $aResult['import-way']) { if ($bUseOSMApi) { - $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full'; + $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/' . $aResult['import-way'] . '/full'; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way(' . $aResult['import-way'] . ');node(w););out%20meta;'; } } if (isset($aResult['import-relation']) && $aResult['import-relation']) { if ($bUseOSMApi) { - $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full'; + $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/' . $aResult['import-relation'] . '/full'; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel(' . $aResult['import-relation'] . ');way(r);node(w));node(r));out%20meta;'; } } @@ -200,8 +180,8 @@ if ($sContentURL) { if ($bHaveDiff) { // import generated change file - $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile; - echo $sCMD."\n"; + $sCMD = $sOsm2pgsqlCmd . ' ' . $sTemporaryFile; + echo $sCMD . "\n"; $iErrorLevel = runWithEnv($sCMD, $aProcEnv); if ($iErrorLevel) { fail("osm2pgsql exited with error level $iErrorLevel\n"); @@ -209,7 +189,7 @@ if ($bHaveDiff) { } if ($aResult['deduplicate']) { - $oDB =& getDB(); + $oDB = &getDB(); if (getPostgresVersion($oDB) < 9.3) { fail('ERROR: deduplicate is only currently supported in postgresql 9.3'); @@ -221,7 +201,7 @@ if ($aResult['deduplicate']) { // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates foreach ($aPartitions as $i => $sPartition) { - $sSQL = 'select count(*) from search_name_'.$sPartition; + $sSQL = 'select count(*) from search_name_' . $sPartition; $nEntries = chksql($oDB->getOne($sSQL)); if ($nEntries == 0) { unset($aPartitions[$i]); @@ -233,11 +213,14 @@ if ($aResult['deduplicate']) { $sSQL .= ' group by word_token having count(*) > 1 order by word_token'; $aDuplicateTokens = chksql($oDB->getAll($sSQL)); foreach ($aDuplicateTokens as $aToken) { - if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue; - echo 'Deduping '.$aToken['word_token']."\n"; + if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') { + continue; + } + + echo 'Deduping ' . $aToken['word_token'] . "\n"; $sSQL = 'select word_id,'; $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num'; - $sSQL .= " from word where word_token = '".$aToken['word_token']; + $sSQL .= " from word where word_token = '" . $aToken['word_token']; $sSQL .= "' and class is null and type is null and country_code is null order by num desc"; $aTokenSet = chksql($oDB->getAll($sSQL)); @@ -246,34 +229,34 @@ if ($aResult['deduplicate']) { foreach ($aTokenSet as $aRemove) { $sSQL = 'update search_name set'; - $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; + $sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . '),'; + $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; + $sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']'; chksql($oDB->query($sSQL)); $sSQL = 'update search_name set'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']'; + $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; + $sSQL .= ' where nameaddress_vector @> ARRAY[' . $aRemove['word_id'] . ']'; chksql($oDB->query($sSQL)); $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; + $sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; + $sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']'; chksql($oDB->query($sSQL)); foreach ($aPartitions as $sPartition) { - $sSQL = 'update search_name_'.$sPartition.' set'; - $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; + $sSQL = 'update search_name_' . $sPartition . ' set'; + $sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; + $sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']'; chksql($oDB->query($sSQL)); $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; - $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; + $sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; + $sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']'; chksql($oDB->query($sSQL)); } - $sSQL = 'delete from word where word_id = '.$aRemove['word_id']; + $sSQL = 'delete from word where word_id = ' . $aRemove['word_id']; chksql($oDB->query($sSQL)); } } @@ -281,12 +264,12 @@ if ($aResult['deduplicate']) { if ($aResult['recompute-word-counts']) { info('Recompute frequency of full-word search terms'); - $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql'); + $sTemplate = file_get_contents(CONST_BasePath . '/sql/words_from_search_name.sql'); runSQLScript($sTemplate, true, true); } if ($aResult['index']) { - $sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']; + $sCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'] . ' -r ' . $aResult['index-rank']; if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { $sCmd .= ' -H ' . $aDSNInfo['hostspec']; } @@ -301,13 +284,13 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) { fail('Error: Update interval too low for download.geofabrik.de. ' . - "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n"); + "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n"); } - $sImportFile = CONST_InstallPath.'/osmosischange.osc'; - $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size; - $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile; - $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances']; + $sImportFile = CONST_InstallPath . '/osmosischange.osc'; + $sCMDDownload = CONST_Pyosmium_Binary . ' --server ' . CONST_Replication_Url . ' -o ' . $sImportFile . ' -s ' . CONST_Replication_Max_Diff_size; + $sCMDImport = $sOsm2pgsqlCmd . ' ' . $sImportFile; + $sCMDIndex = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances']; if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { $sCMDIndex .= ' -H ' . $aDSNInfo['hostspec']; } @@ -324,7 +307,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { exit(1); } - echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n"; + echo 'Currently at sequence ' . $aLastState['sequence_id'] . ' (' . $aLastState['lastimportdate'] . ') - ' . $aLastState['indexed'] . " indexed\n"; $sBatchEnd = $aLastState['lastimportdate']; $iEndSequence = $aLastState['sequence_id']; @@ -347,23 +330,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { if (file_exists($sImportFile)) { unlink($sImportFile); } - exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult); + exec($sCMDDownload . ' -I ' . $iNextSeq, $aOutput, $iResult); if ($iResult == 3) { - echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n"; + echo 'No new updates. Sleeping for ' . CONST_Replication_Recheck_Interval . " sec.\n"; sleep(CONST_Replication_Recheck_Interval); } elseif ($iResult != 0) { echo 'ERROR: updates failed.'; exit($iResult); } else { - $iEndSequence = (int)$aOutput[0]; + $iEndSequence = (int) $aOutput[0]; } } while ($iResult); // get the newest object from the diff file $sBatchEnd = 0; $iRet = 0; - exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet); + exec(CONST_BasePath . '/utils/osm_file_date.py ' . $sImportFile, $sBatchEnd, $iRet); if ($iRet == 5) { echo "Diff file is empty. skipping import.\n"; if (!$aResult['import-osmosis-all']) { @@ -379,7 +362,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // Import the file $fCMDStartTime = time(); - echo $sCMDImport."\n"; + echo $sCMDImport . "\n"; unset($sJunk); $iErrorLevel = runWithEnv($sCMDImport, $aProcEnv); if ($iErrorLevel) { @@ -392,8 +375,8 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = 'INSERT INTO import_osmosis_log'; $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)'; $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'"; - $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','"; - $sSQL .= date('Y-m-d H:i:s')."','import')"; + $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','"; + $sSQL .= date('Y-m-d H:i:s') . "','import')"; var_Dump($sSQL); chksql($oDB->query($sSQL)); @@ -401,7 +384,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence"; var_Dump($sSQL); chksql($oDB->query($sSQL)); - echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; + echo date('Y-m-d H:i:s') . " Completed download step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n"; } // Index file @@ -419,18 +402,21 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = 'INSERT INTO import_osmosis_log'; $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)'; $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'"; - $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','"; - $sSQL .= date('Y-m-d H:i:s')."','index')"; + $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','"; + $sSQL .= date('Y-m-d H:i:s') . "','index')"; var_Dump($sSQL); $oDB->query($sSQL); - echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; + echo date('Y-m-d H:i:s') . " Completed index step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n"; $sSQL = 'update import_status set indexed = true'; $oDB->query($sSQL); } $fDuration = time() - $fStartTime; - echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n"; - if (!$aResult['import-osmosis-all']) exit(0); + echo date('Y-m-d H:i:s') . " Completed all for $sBatchEnd in " . round($fDuration / 60, 2) . " minutes\n"; + if (!$aResult['import-osmosis-all']) { + exit(0); + } + } } From 42e79bfab9f378b70583ae023e5f19678804f1ac Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Fri, 31 Aug 2018 21:44:49 +0200 Subject: [PATCH 05/12] delete an empty line to make the pendantic Mister Travis happy --- utils/update.php | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/update.php b/utils/update.php index 176bde06..172e71e2 100755 --- a/utils/update.php +++ b/utils/update.php @@ -417,6 +417,5 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { if (!$aResult['import-osmosis-all']) { exit(0); } - } } From aa6ac5a75130c066e2b46af0a12d3fe71e614d1b Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Fri, 31 Aug 2018 22:01:53 +0200 Subject: [PATCH 06/12] more format changes for Mr. Travis --- lib/SetupClass.php | 404 +++++++++++++++++---------------------------- 1 file changed, 147 insertions(+), 257 deletions(-) diff --git a/lib/SetupClass.php b/lib/SetupClass.php index d50d9905..794377dc 100644 --- a/lib/SetupClass.php +++ b/lib/SetupClass.php @@ -4,28 +4,29 @@ namespace Nominatim\Setup; class SetupFunctions { - protected $iCacheMemory; // set in constructor - protected $iInstances; // set in constructor - protected $sModulePath; // set in constructor - protected $aDSNInfo; // set in constructor = DB::parseDSN(CONST_Database_DSN); - protected $sVerbose; // set in constructor - protected $sIgnoreErrors; // set in constructor - protected $bEnableDiffUpdates; // set in constructor - protected $bEnableDebugStatements; // set in constructor - protected $bNoPartitions; // set in constructor - protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData - // pgsqlRunPartitionScript, calculatePostcodes, ..if no already set + protected $iCacheMemory; // set in constructor + protected $iInstances; // set in constructor + protected $sModulePath; // set in constructor + protected $aDSNInfo; // set in constructor = DB::parseDSN(CONST_Database_DSN); + protected $sVerbose; // set in constructor + protected $sIgnoreErrors; // set in constructor + protected $bEnableDiffUpdates; // set in constructor + protected $bEnableDebugStatements; // set in constructor + protected $bNoPartitions; // set in constructor + protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData + // pgsqlRunPartitionScript, calculatePostcodes, ..if no already set public function __construct($aCMDResult) { // by default, use all but one processor, but never more than 15. $this->iInstances = isset($aCMDResult['threads']) - ? $aCMDResult['threads'] - : (min(16, getProcessorCount()) - 1); + ? $aCMDResult['threads'] + : (min(16, getProcessorCount()) - 1); + if ($this->iInstances < 1) { $this->iInstances = 1; - warn('resetting threads to ' . $this->iInstances); + warn('resetting threads to '.$this->iInstances); } // Assume we can steal all the cache memory in the box (unless told otherwise) @@ -37,13 +38,11 @@ class SetupFunctions $this->sModulePath = CONST_Database_Module_Path; info('module path: ' . $this->sModulePath); - + // prepares DB for import or update, sets the Data Source Name $this->aDSNInfo = \DB::parseDSN(CONST_Database_DSN); - if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) { - $this->aDSNInfo['port'] = 5432; - } - + if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; + // setting member variables based on command line options stored in $aCMDResult $this->sVerbose = $aCMDResult['verbose']; $this->sIgnoreErrors = $aCMDResult['ignore-errors']; @@ -57,10 +56,10 @@ class SetupFunctions info('Create DB'); $sDB = \DB::connect(CONST_Database_DSN, false); if (!\PEAR::isError($sDB)) { - fail('database already exists (' . CONST_Database_DSN . ')'); + fail('database already exists ('.CONST_Database_DSN.')'); } - $sCreateDBCmd = 'createdb -E UTF-8 -p ' . $this->aDSNInfo['port'] . ' ' . $this->aDSNInfo['database']; + $sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database']; if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { $sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username']; } @@ -75,19 +74,16 @@ class SetupFunctions } $result = runWithEnv($sCreateDBCmd, $aProcEnv); - if ($result != 0) { - fail('Error executing external command: ' . $sCreateDBCmd); - } - + if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); } public function setupDB() { info('Setup DB'); - $this->oDB = &getDB(); + $this->oDB =& getDB(); $fPostgresVersion = getPostgresVersion($this->oDB); - echo 'Postgres version found: ' . $fPostgresVersion . "\n"; + echo 'Postgres version found: '.$fPostgresVersion."\n"; if ($fPostgresVersion < 9.1) { fail('Minimum supported version of Postgresql is 9.1.'); @@ -106,8 +102,9 @@ class SetupFunctions warn('Postgresql is too old. extratags and namedetails API not available.'); } + $fPostgisVersion = getPostgisVersion($this->oDB); - echo 'Postgis version found: ' . $fPostgisVersion . "\n"; + echo 'Postgis version found: '.$fPostgisVersion."\n"; if ($fPostgisVersion < 2.1) { // Functions were renamed in 2.1 and throw an annoying deprecation warning @@ -118,31 +115,32 @@ class SetupFunctions $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); } - $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '" . CONST_Database_Web_User . "'")); + $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'")); if ($i == 0) { - echo "\nERROR: Web user '" . CONST_Database_Web_User . "' does not exist. Create it with:\n"; - echo "\n createuser " . CONST_Database_Web_User . "\n\n"; + echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; + echo "\n createuser ".CONST_Database_Web_User."\n\n"; exit(1); } - if (!file_exists(CONST_ExtraDataPath . '/country_osm_grid.sql.gz')) { + if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { echo 'Error: you need to download the country_osm_grid first:'; - echo "\n wget -O " . CONST_ExtraDataPath . "/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; + echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; exit(1); } - $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_name.sql'); - $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_naturalearthdata.sql'); - $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_osm_grid.sql.gz'); - $this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_table.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql'); - if (file_exists(CONST_BasePath . '/data/gb_postcode_data.sql.gz')) { - $this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_data.sql.gz'); + + if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz'); } else { warn('external UK postcode table not found.'); } if (CONST_Use_Extra_US_Postcodes) { - $this->pgsqlRunScriptFile(CONST_BasePath . '/data/us_postcode.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); } if ($this->bNoPartitions) { @@ -168,29 +166,23 @@ class SetupFunctions fail("osm2pgsql not found in '$osm2pgsql'"); } + + if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $osm2pgsql .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File; - } - - if (CONST_Tablespace_Osm2pgsql_Data) { - $osm2pgsql .= ' --tablespace-slim-data ' . CONST_Tablespace_Osm2pgsql_Data; - } - - if (CONST_Tablespace_Osm2pgsql_Index) { - $osm2pgsql .= ' --tablespace-slim-index ' . CONST_Tablespace_Osm2pgsql_Index; - } - - if (CONST_Tablespace_Place_Data) { - $osm2pgsql .= ' --tablespace-main-data ' . CONST_Tablespace_Place_Data; - } - - if (CONST_Tablespace_Place_Index) { - $osm2pgsql .= ' --tablespace-main-index ' . CONST_Tablespace_Place_Index; + $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; } + if (CONST_Tablespace_Osm2pgsql_Data) + $osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data; + if (CONST_Tablespace_Osm2pgsql_Index) + $osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index; + if (CONST_Tablespace_Place_Data) + $osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data; + if (CONST_Tablespace_Place_Index) + $osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index; $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; - $osm2pgsql .= ' -C ' . $this->iCacheMemory; - $osm2pgsql .= ' -P ' . $this->aDSNInfo['port']; + $osm2pgsql .= ' -C '.$this->iCacheMemory; + $osm2pgsql .= ' -P '.$this->aDSNInfo['port']; if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { $osm2pgsql .= ' -U ' . $this->aDSNInfo['username']; } @@ -201,12 +193,9 @@ class SetupFunctions if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); } - $osm2pgsql .= ' -d ' . $this->aDSNInfo['database'] . ' ' . $sOSMFile; + $osm2pgsql .= ' -d '.$this->aDSNInfo['database'].' '.$sOSMFile; runWithEnv($osm2pgsql, $aProcEnv); - if ($this->oDB == null) { - $this->oDB = &getDB(); - } - + if ($this->oDB == null) $this->oDB =& getDB(); if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) { fail('No Data'); } @@ -223,7 +212,7 @@ class SetupFunctions { info('Create Tables'); - $sTemplate = file_get_contents(CONST_BasePath . '/sql/tables.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = $this->replaceTablespace( '{ts:address-data}', @@ -263,7 +252,7 @@ class SetupFunctions { info('Create Partition Tables'); - $sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-tables.src.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); $sTemplate = $this->replaceTablespace( '{ts:address-data}', CONST_Tablespace_Address_Data, @@ -307,14 +296,14 @@ class SetupFunctions { info('Create Partition Functions'); - $sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-functions.src.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); $this->pgsqlRunPartitionScript($sTemplate); } public function importWikipediaArticles() { - $sWikiArticlesFile = CONST_Wikipedia_Data_Path . '/wikipedia_article.sql.bin'; - $sWikiRedirectsFile = CONST_Wikipedia_Data_Path . '/wikipedia_redirect.sql.bin'; + $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; + $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; if (file_exists($sWikiArticlesFile)) { info('Importing wikipedia articles'); $this->pgsqlRunDropAndRestore($sWikiArticlesFile); @@ -334,78 +323,40 @@ class SetupFunctions { info('Drop old Data'); - if ($this->oDB == null) { - $this->oDB = &getDB(); - } - - if (!pg_query($this->oDB->connection, 'TRUNCATE word')) { - fail(pg_last_error($this->oDB->connection)); - } + if ($this->oDB == null) $this->oDB =& getDB(); + if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection)); echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection)); echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection)); echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection)); echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection)); echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection)); echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection)); echo '.'; - if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection)); echo '.'; - if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection)); echo '.'; - if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection)); echo '.'; $sSQL = 'select distinct partition from country_name'; $aPartitions = chksql($this->oDB->getCol($sSQL)); - if (!$this->bNoPartitions) { - $aPartitions[] = 0; - } - + if (!$this->bNoPartitions) $aPartitions[] = 0; foreach ($aPartitions as $sPartition) { - if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_' . $sPartition)) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection)); echo '.'; } // used by getorcreate_word_id to ignore frequent partial words $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; - $sSQL .= '$$ SELECT ' . CONST_Max_Word_Frequency . ' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; + $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; if (!pg_query($this->oDB->connection, $sSQL)) { fail(pg_last_error($this->oDB->connection)); } @@ -414,7 +365,7 @@ class SetupFunctions // pre-create the word list if (!$bDisableTokenPrecalc) { info('Loading word list'); - $this->pgsqlRunScriptFile(CONST_BasePath . '/data/words.sql'); + $this->pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql'); } info('Load Data'); @@ -422,30 +373,24 @@ class SetupFunctions $aDBInstances = array(); $iLoadThreads = max(1, $this->iInstances - 1); for ($i = 0; $i < $iLoadThreads; $i++) { - $aDBInstances[$i] = &getDB(true); + $aDBInstances[$i] =& getDB(true); $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; $sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; $sSQL .= ' and ST_IsValid(geometry)'; - if ($this->sVerbose) { - echo "$sSQL\n"; - } - + if ($this->sVerbose) echo "$sSQL\n"; if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { fail(pg_last_error($aDBInstances[$i]->connection)); } } // last thread for interpolation lines - $aDBInstances[$iLoadThreads] = &getDB(true); + $aDBInstances[$iLoadThreads] =& getDB(true); $sSQL = 'insert into location_property_osmline'; $sSQL .= ' (osm_id, address, linegeo)'; $sSQL .= ' SELECT osm_id, address, geometry from place where '; $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; - if ($this->sVerbose) { - echo "$sSQL\n"; - } - + if ($this->sVerbose) echo "$sSQL\n"; if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); } @@ -477,7 +422,7 @@ class SetupFunctions if ($sDatabaseDate === false) { warn('could not determine database date.'); } else { - $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('" . $sDatabaseDate . "')"; + $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; pg_query($this->oDB->connection, $sSQL); echo "Latest data imported from $sDatabaseDate.\n"; } @@ -487,7 +432,7 @@ class SetupFunctions { info('Import Tiger data'); - $sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_start.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = $this->replaceTablespace( '{ts:aux-data}', @@ -503,11 +448,11 @@ class SetupFunctions $aDBInstances = array(); for ($i = 0; $i < $this->iInstances; $i++) { - $aDBInstances[$i] = &getDB(true); + $aDBInstances[$i] =& getDB(true); } - foreach (glob(CONST_Tiger_Data_Path . '/*.sql') as $sFile) { - echo $sFile . ': '; + foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) { + echo $sFile.': '; $hFile = fopen($sFile, 'r'); $sSQL = fgets($hFile, 100000); $iLines = 0; @@ -516,14 +461,8 @@ class SetupFunctions if (!pg_connection_busy($aDBInstances[$i]->connection)) { while (pg_get_result($aDBInstances[$i]->connection)); $sSQL = fgets($hFile, 100000); - if (!$sSQL) { - break 2; - } - - if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!$sSQL) break 2; + if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); $iLines++; if ($iLines == 1000) { echo '.'; @@ -539,10 +478,7 @@ class SetupFunctions while ($bAnyBusy) { $bAnyBusy = false; for ($i = 0; $i < $this->iInstances; $i++) { - if (pg_connection_busy($aDBInstances[$i]->connection)) { - $bAnyBusy = true; - } - + if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; } usleep(10); } @@ -550,7 +486,7 @@ class SetupFunctions } info('Creating indexes on Tiger data'); - $sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_finish.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = $this->replaceTablespace( '{ts:aux-data}', @@ -568,15 +504,13 @@ class SetupFunctions public function calculatePostcodes($bCMDResultAll) { info('Calculate Postcodes'); - if ($this->oDB == null) { - $this->oDB = &getDB(); - } - + if ($this->oDB == null) $this->oDB =& getDB(); if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { fail(pg_last_error($this->oDB->connection)); } - $sSQL = 'INSERT INTO location_postcode'; + + $sSQL = 'INSERT INTO location_postcode'; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; $sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; @@ -592,29 +526,24 @@ class SetupFunctions if (CONST_Use_Extra_US_Postcodes) { // only add postcodes that are not yet available in OSM - $sSQL = 'INSERT INTO location_postcode'; + $sSQL = 'INSERT INTO location_postcode'; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; $sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; $sSQL .= ' (SELECT postcode FROM location_postcode'; $sSQL .= " WHERE country_code = 'us')"; - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); } // add missing postcodes for GB (if available) - $sSQL = 'INSERT INTO location_postcode'; + $sSQL = 'INSERT INTO location_postcode'; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; $sSQL .= ' (SELECT postcode FROM location_postcode'; $sSQL .= " WHERE country_code = 'gb')"; - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } + if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); if (!$bCMDResultAll) { $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; @@ -634,8 +563,8 @@ class SetupFunctions public function index($bIndexNoanalyse) { $sOutputFile = ''; - $sBaseCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $this->aDSNInfo['database'] . ' -P ' - . $this->aDSNInfo['port'] . ' -t ' . $this->iInstances . $sOutputFile; + $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P ' + .$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile; if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; } @@ -648,45 +577,33 @@ class SetupFunctions } info('Index ranks 0 - 4'); - $iStatus = runWithEnv($sBaseCmd . ' -R 4', $aProcEnv); + $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } - if (!$bIndexNoanalyse) { - $this->pgsqlRunScript('ANALYSE'); - } - + if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); info('Index ranks 5 - 25'); - $iStatus = runWithEnv($sBaseCmd . ' -r 5 -R 25', $aProcEnv); + $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } - if (!$bIndexNoanalyse) { - $this->pgsqlRunScript('ANALYSE'); - } - + if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); info('Index ranks 26 - 30'); - $iStatus = runWithEnv($sBaseCmd . ' -r 26', $aProcEnv); + $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } info('Index postcodes'); - if ($this->oDB == null) { - $this->oDB = &getDB(); - } - + if ($this->oDB == null) $this->oDB =& getDB(); $sSQL = 'UPDATE location_postcode SET indexed_status = 0'; - if (!pg_query($this->oDB->connection, $sSQL)) { - fail(pg_last_error($this->oDB->connection)); - } - + if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); } public function createSearchIndices() { info('Create Search indices'); - $sTemplate = file_get_contents(CONST_BasePath . '/sql/indices.src.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = $this->replaceTablespace( '{ts:address-index}', @@ -715,12 +632,12 @@ class SetupFunctions $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),' - . 'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; + .'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; if (CONST_Languages) { $sSQL .= 'in '; $sDelim = '('; foreach (explode(',', CONST_Languages) as $sLang) { - $sSQL .= $sDelim . "'name:$sLang'"; + $sSQL .= $sDelim."'name:$sLang'"; $sDelim = ','; } $sSQL .= ')'; @@ -742,27 +659,24 @@ class SetupFunctions // created. USE AT YOUR OWN PERIL. // tables we want to keep. everything else goes. $aKeepTables = array( - '*columns', - 'import_polygon_*', - 'import_status', - 'place_addressline', - 'location_postcode', - 'location_property*', - 'placex', - 'search_name', - 'seq_*', - 'word', - 'query_log', - 'new_query_log', - 'spatial_ref_sys', - 'country_name', - 'place_classtype_*', - ); - - if ($this->oDB = null) { - $this->oDB = &getDB(); - } + '*columns', + 'import_polygon_*', + 'import_status', + 'place_addressline', + 'location_postcode', + 'location_property*', + 'placex', + 'search_name', + 'seq_*', + 'word', + 'query_log', + 'new_query_log', + 'spatial_ref_sys', + 'country_name', + 'place_classtype_*' + ); + if ($this->oDB = null) $this->oDB =& getDB(); $aDropTables = array(); $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); @@ -774,37 +688,25 @@ class SetupFunctions break; } } - if (!$bFound) { - array_push($aDropTables, $sTable); - } - + if (!$bFound) array_push($aDropTables, $sTable); } foreach ($aDropTables as $sDrop) { - if ($this->sVerbose) { - echo "dropping table $sDrop\n"; - } - + if ($this->sVerbose) echo "dropping table $sDrop\n"; @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE"); // ignore warnings/errors as they might be caused by a table having // been deleted already by CASCADE } if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - if ($sVerbose) { - echo 'deleting ' . CONST_Osm2pgsql_Flatnode_File . "\n"; - } - + if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; unlink(CONST_Osm2pgsql_Flatnode_File); } } private function pgsqlRunDropAndRestore($sDumpFile) { - if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) { - $this->aDSNInfo['port'] = 5432; - } - - $sCMD = 'pg_restore -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database'] . ' -Fc --clean ' . $sDumpFile; + if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; + $sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile; if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; } @@ -815,9 +717,9 @@ class SetupFunctions if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); } - $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" + $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" } - + private function pgsqlRunScript($sScript, $bfatal = true) { runSQLScript( @@ -830,7 +732,7 @@ class SetupFunctions private function createSqlFunctions() { - $sTemplate = file_get_contents(CONST_BasePath . '/sql/functions.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); if ($this->bEnableDiffUpdates) { $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); @@ -852,15 +754,11 @@ class SetupFunctions private function pgsqlRunPartitionScript($sTemplate) { - if ($this->oDB == null) { - $this->oDB = &getDB(); - } + if ($this->oDB == null) $this->oDB =& getDB(); $sSQL = 'select distinct partition from country_name'; $aPartitions = chksql($this->oDB->getCol($sSQL)); - if (!$this->bNoPartitions) { - $aPartitions[] = 0; - } + if (!$this->bNoPartitions) $aPartitions[] = 0; preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); foreach ($aMatches as $aMatch) { @@ -876,11 +774,9 @@ class SetupFunctions private function pgsqlRunScriptFile($sFilename) { - if (!file_exists($sFilename)) { - fail('unable to find ' . $sFilename); - } + if (!file_exists($sFilename)) fail('unable to find '.$sFilename); - $sCMD = 'psql -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database']; + $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database']; if (!$this->sVerbose) { $sCMD .= ' -q'; } @@ -897,32 +793,26 @@ class SetupFunctions $ahGzipPipes = null; if (preg_match('/\\.gz$/', $sFilename)) { $aDescriptors = array( - 0 => array('pipe', 'r'), - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a'), - ); - $hGzipProcess = proc_open('zcat ' . $sFilename, $aDescriptors, $ahGzipPipes); - if (!is_resource($hGzipProcess)) { - fail('unable to start zcat'); - } - + 0 => array('pipe', 'r'), + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a') + ); + $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); + if (!is_resource($hGzipProcess)) fail('unable to start zcat'); $aReadPipe = $ahGzipPipes[1]; fclose($ahGzipPipes[0]); } else { - $sCMD .= ' -f ' . $sFilename; + $sCMD .= ' -f '.$sFilename; $aReadPipe = array('pipe', 'r'); } $aDescriptors = array( - 0 => $aReadPipe, - 1 => array('pipe', 'w'), - 2 => array('file', '/dev/null', 'a'), - ); + 0 => $aReadPipe, + 1 => array('pipe', 'w'), + 2 => array('file', '/dev/null', 'a') + ); $ahPipes = null; $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); - if (!is_resource($hProcess)) { - fail('unable to start pgsql'); - } - + if (!is_resource($hProcess)) fail('unable to start pgsql'); // TODO: error checking while (!feof($ahPipes[1])) { echo fread($ahPipes[1], 4096); @@ -941,7 +831,7 @@ class SetupFunctions private function replaceTablespace($sTemplate, $sTablespace, $sSql) { if ($sTablespace) { - $sSql = str_replace($sTemplate, 'TABLESPACE "' . $sTablespace . '"', $sSql); + $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); } else { $sSql = str_replace($sTemplate, '', $sSql); } From d10f63b6667b6d55298bee3848ce5c3c03e131fb Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Wed, 5 Sep 2018 22:01:03 +0200 Subject: [PATCH 07/12] format change revert, removed bogus CL options, SetupClass to a new dir --- lib/{ => classes}/SetupClass.php | 42 ++++++--- lib/setup_functions.php | 3 - utils/setup.php | 10 +- utils/update.php | 157 +++++++++++++++---------------- 4 files changed, 106 insertions(+), 106 deletions(-) rename lib/{ => classes}/SetupClass.php (96%) mode change 100644 => 100755 diff --git a/lib/SetupClass.php b/lib/classes/SetupClass.php old mode 100644 new mode 100755 similarity index 96% rename from lib/SetupClass.php rename to lib/classes/SetupClass.php index 794377dc..120f625e --- a/lib/SetupClass.php +++ b/lib/classes/SetupClass.php @@ -45,10 +45,24 @@ class SetupFunctions // setting member variables based on command line options stored in $aCMDResult $this->sVerbose = $aCMDResult['verbose']; - $this->sIgnoreErrors = $aCMDResult['ignore-errors']; $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; - $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; - $this->bNoPartitions = $aCMDResult['no-partitions']; + + //setting default values which are not set by the update.php array + if (isset($aCMDResult['ignore-errors'])) { + $this->sIgnoreErrors = $aCMDResult['ignore-errors']; + } else { + $this->sIgnoreErrors = false; + } + if (isset($aCMDResult['enable-debug-statements'])) { + $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; + } else { + $this->bEnableDebugStatements = false; + } + if (isset($aCMDResult['no-partitions'])) { + $this->bNoPartitions = $aCMDResult['no-partitions']; + } else { + $this->bNoPartitions = false; + } } public function createDB() @@ -61,11 +75,11 @@ class SetupFunctions $sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database']; if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username']; + $sCreateDBCmd .= ' -U '.$this->aDSNInfo['username']; } if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sCreateDBCmd .= ' -h ' . $this->aDSNInfo['hostspec']; + $sCreateDBCmd .= ' -h '.$this->aDSNInfo['hostspec']; } $aProcEnv = null; @@ -184,10 +198,10 @@ class SetupFunctions $osm2pgsql .= ' -C '.$this->iCacheMemory; $osm2pgsql .= ' -P '.$this->aDSNInfo['port']; if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $osm2pgsql .= ' -U ' . $this->aDSNInfo['username']; + $osm2pgsql .= ' -U '.$this->aDSNInfo['username']; } if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $osm2pgsql .= ' -H ' . $this->aDSNInfo['hostspec']; + $osm2pgsql .= ' -H '.$this->aDSNInfo['hostspec']; } $aProcEnv = null; if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { @@ -402,7 +416,7 @@ class SetupFunctions // PGSQL_EMPTY_QUERY, PGSQL_COMMAND_OK, PGSQL_TUPLES_OK, // PGSQL_COPY_OUT, PGSQL_COPY_IN, PGSQL_BAD_RESPONSE, // PGSQL_NONFATAL_ERROR and PGSQL_FATAL_ERROR - echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; + // echo 'Query result ' . $i . ' is: ' . $resultStatus . "\n"; if ($resultStatus != PGSQL_COMMAND_OK && $resultStatus != PGSQL_TUPLES_OK) { $resultError = pg_result_error($hPGresult); echo '-- error text ' . $i . ': ' . $resultError . "\n"; @@ -566,10 +580,10 @@ class SetupFunctions $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P ' .$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile; if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; + $sBaseCmd .= ' -H '.$this->aDSNInfo['hostspec']; } if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sBaseCmd .= ' -U ' . $this->aDSNInfo['username']; + $sBaseCmd .= ' -U '.$this->aDSNInfo['username']; } $aProcEnv = null; if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { @@ -708,10 +722,10 @@ class SetupFunctions if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; $sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile; if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; + $sCMD .= ' -h '.$this->aDSNInfo['hostspec']; } if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sCMD .= ' -U ' . $this->aDSNInfo['username']; + $sCMD .= ' -U '.$this->aDSNInfo['username']; } $aProcEnv = null; if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { @@ -781,10 +795,10 @@ class SetupFunctions $sCMD .= ' -q'; } if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { - $sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; + $sCMD .= ' -h '.$this->aDSNInfo['hostspec']; } if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { - $sCMD .= ' -U ' . $this->aDSNInfo['username']; + $sCMD .= ' -U '.$this->aDSNInfo['username']; } $aProcEnv = null; if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { diff --git a/lib/setup_functions.php b/lib/setup_functions.php index 7b705168..af718ed0 100755 --- a/lib/setup_functions.php +++ b/lib/setup_functions.php @@ -118,9 +118,6 @@ function createUpdateArgvArray() array('create-functions', '', 0, 1, 1, 1, 'bool', 'Create functions'), array('enable-diff-updates', '', 0, 1, 1, 1, 'bool', 'Turn on the code required to make diff updates work'), - array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), - array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), - array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), ); return $aCMDOptions; } diff --git a/utils/setup.php b/utils/setup.php index 989d28e1..63c4d211 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -1,13 +1,12 @@ #!@PHP_BIN@ -Cq indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt // ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB() - -require_once CONST_BasePath . '/lib/setup_functions.php'; +require_once(CONST_BasePath.'/lib/setup_functions.php'); ini_set('memory_limit', '800M'); use Nominatim\Setup\SetupFunctions as SetupFunctions; @@ -25,7 +24,6 @@ $bDidSomething = false; if ($aCMDResult['import-data'] || $aCMDResult['all']) { // to remain in /lib/setup_functions.php function checkInFile($aCMDResult['osm-file']); - echo $aCMDResult['osm-file']; } // osmosis init is no longer supported diff --git a/utils/update.php b/utils/update.php index 172e71e2..3f9bf93a 100755 --- a/utils/update.php +++ b/utils/update.php @@ -1,42 +1,35 @@ #!@PHP_BIN@ -Cq getTotalMemoryMB()) { $iCacheMemory = getCacheMemoryMB(); echo "WARNING: resetting cache memory to $iCacheMemory\n"; } -$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary . ' -klas --number-processes 1 -C ' . $iCacheMemory . ' -O gazetteer -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port']; +$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { $sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username']; } @@ -49,12 +42,12 @@ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { } if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { - $sOsm2pgsqlCmd .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File; + $sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; } if ($aResult['init-updates']) { // sanity check that the replication URL is correct - $sBaseState = file_get_contents(CONST_Replication_Url . '/state.txt'); + $sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt'); if ($sBaseState === false) { echo "\nCannot find state.txt file at the configured replication URL.\n"; echo "Does the URL point to a directory containing OSM update data?\n\n"; @@ -68,7 +61,7 @@ if ($aResult['init-updates']) { fail('CONST_Pyosmium_Binary not configured'); } $aOutput = 0; - $sCmd = CONST_Pyosmium_Binary . ' --help'; + $sCmd = CONST_Pyosmium_Binary.' --help'; exec($sCmd, $aOutput, $iRet); if ($iRet != 0) { echo "Cannot execute pyosmium-get-changes.\n"; @@ -78,8 +71,11 @@ if ($aResult['init-updates']) { } if (!$aResult['no-update-functions']) { - // instatiate setupClass to use the function therein - // instantiate Setup class + // Try accessing the C module, + if (!checkModulePresence()) { + fail('error loading nominatim.so module'); + } + // instantiate setupClass to use the function therein $cSetup = new SetupFunctions($aResult); $cSetup->createFunctions(); } @@ -88,11 +84,11 @@ if ($aResult['init-updates']) { if ($sDatabaseDate === false) { fail('Cannot determine date of database.'); } - $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3 * 60 * 60)); + $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60)); // get the appropriate state id $aOutput = 0; - $sCmd = CONST_Pyosmium_Binary . ' -D ' . $sWindBack . ' --server ' . CONST_Replication_Url; + $sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url; exec($sCmd, $aOutput, $iRet); if ($iRet != 0 || $aOutput[0] == 'None') { fail('Error running pyosmium tools'); @@ -100,7 +96,7 @@ if ($aResult['init-updates']) { pg_query($oDB->connection, 'TRUNCATE import_status'); $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('"; - $sSQL .= $sDatabaseDate . "'," . $aOutput[0] . ', true)'; + $sSQL .= $sDatabaseDate."',".$aOutput[0].', true)'; if (!pg_query($oDB->connection, $sSQL)) { fail('Could not enter sequence into database.'); } @@ -115,7 +111,7 @@ if ($aResult['check-for-updates']) { fail('Updates not set up. Please run ./utils/update.php --init-updates.'); } - system(CONST_BasePath . '/utils/check_server_for_updates.py ' . CONST_Replication_Url . ' ' . $aLastState['sequence_id'], $iRet); + system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet); exit($iRet); } @@ -128,8 +124,8 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { } // Import the file - $sCMD = $sOsm2pgsqlCmd . ' ' . $sNextFile; - echo $sCMD . "\n"; + $sCMD = $sOsm2pgsqlCmd.' '.$sNextFile; + echo $sCMD."\n"; $iErrorLevel = runWithEnv($sCMD, $aProcEnv); if ($iErrorLevel) { @@ -141,35 +137,35 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) { if ($aResult['calculate-postcodes']) { info('Update postcodes centroids'); - $sTemplate = file_get_contents(CONST_BasePath . '/sql/update-postcodes.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql'); runSQLScript($sTemplate, true, true); } -$sTemporaryFile = CONST_BasePath . '/data/osmosischange.osc'; +$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc'; $bHaveDiff = false; $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api']; $sContentURL = ''; if (isset($aResult['import-node']) && $aResult['import-node']) { if ($bUseOSMApi) { - $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/' . $aResult['import-node']; + $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node']; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=node(' . $aResult['import-node'] . ');out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;'; } } if (isset($aResult['import-way']) && $aResult['import-way']) { if ($bUseOSMApi) { - $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/' . $aResult['import-way'] . '/full'; + $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full'; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way(' . $aResult['import-way'] . ');node(w););out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;'; } } if (isset($aResult['import-relation']) && $aResult['import-relation']) { if ($bUseOSMApi) { - $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/' . $aResult['import-relation'] . '/full'; + $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full'; } else { - $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel(' . $aResult['import-relation'] . ');way(r);node(w));node(r));out%20meta;'; + $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;'; } } @@ -180,8 +176,8 @@ if ($sContentURL) { if ($bHaveDiff) { // import generated change file - $sCMD = $sOsm2pgsqlCmd . ' ' . $sTemporaryFile; - echo $sCMD . "\n"; + $sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile; + echo $sCMD."\n"; $iErrorLevel = runWithEnv($sCMD, $aProcEnv); if ($iErrorLevel) { fail("osm2pgsql exited with error level $iErrorLevel\n"); @@ -189,7 +185,7 @@ if ($bHaveDiff) { } if ($aResult['deduplicate']) { - $oDB = &getDB(); + $oDB =& getDB(); if (getPostgresVersion($oDB) < 9.3) { fail('ERROR: deduplicate is only currently supported in postgresql 9.3'); @@ -201,7 +197,7 @@ if ($aResult['deduplicate']) { // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates foreach ($aPartitions as $i => $sPartition) { - $sSQL = 'select count(*) from search_name_' . $sPartition; + $sSQL = 'select count(*) from search_name_'.$sPartition; $nEntries = chksql($oDB->getOne($sSQL)); if ($nEntries == 0) { unset($aPartitions[$i]); @@ -213,14 +209,11 @@ if ($aResult['deduplicate']) { $sSQL .= ' group by word_token having count(*) > 1 order by word_token'; $aDuplicateTokens = chksql($oDB->getAll($sSQL)); foreach ($aDuplicateTokens as $aToken) { - if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') { - continue; - } - - echo 'Deduping ' . $aToken['word_token'] . "\n"; + if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue; + echo 'Deduping '.$aToken['word_token']."\n"; $sSQL = 'select word_id,'; $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num'; - $sSQL .= " from word where word_token = '" . $aToken['word_token']; + $sSQL .= " from word where word_token = '".$aToken['word_token']; $sSQL .= "' and class is null and type is null and country_code is null order by num desc"; $aTokenSet = chksql($oDB->getAll($sSQL)); @@ -229,34 +222,34 @@ if ($aResult['deduplicate']) { foreach ($aTokenSet as $aRemove) { $sSQL = 'update search_name set'; - $sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . '),'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; - $sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']'; + $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),'; + $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; + $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; chksql($oDB->query($sSQL)); $sSQL = 'update search_name set'; - $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; - $sSQL .= ' where nameaddress_vector @> ARRAY[' . $aRemove['word_id'] . ']'; + $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; + $sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']'; chksql($oDB->query($sSQL)); $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; - $sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']'; + $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; + $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; chksql($oDB->query($sSQL)); foreach ($aPartitions as $sPartition) { - $sSQL = 'update search_name_' . $sPartition . ' set'; - $sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; - $sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']'; + $sSQL = 'update search_name_'.$sPartition.' set'; + $sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')'; + $sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; chksql($oDB->query($sSQL)); $sSQL = 'update location_area_country set'; - $sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')'; - $sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']'; + $sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; + $sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; chksql($oDB->query($sSQL)); } - $sSQL = 'delete from word where word_id = ' . $aRemove['word_id']; + $sSQL = 'delete from word where word_id = '.$aRemove['word_id']; chksql($oDB->query($sSQL)); } } @@ -264,12 +257,12 @@ if ($aResult['deduplicate']) { if ($aResult['recompute-word-counts']) { info('Recompute frequency of full-word search terms'); - $sTemplate = file_get_contents(CONST_BasePath . '/sql/words_from_search_name.sql'); + $sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql'); runSQLScript($sTemplate, true, true); } if ($aResult['index']) { - $sCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'] . ' -r ' . $aResult['index-rank']; + $sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']; if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { $sCmd .= ' -H ' . $aDSNInfo['hostspec']; } @@ -284,13 +277,13 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) { fail('Error: Update interval too low for download.geofabrik.de. ' . - "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n"); + "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n"); } - $sImportFile = CONST_InstallPath . '/osmosischange.osc'; - $sCMDDownload = CONST_Pyosmium_Binary . ' --server ' . CONST_Replication_Url . ' -o ' . $sImportFile . ' -s ' . CONST_Replication_Max_Diff_size; - $sCMDImport = $sOsm2pgsqlCmd . ' ' . $sImportFile; - $sCMDIndex = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances']; + $sImportFile = CONST_InstallPath.'/osmosischange.osc'; + $sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size; + $sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile; + $sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances']; if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { $sCMDIndex .= ' -H ' . $aDSNInfo['hostspec']; } @@ -307,7 +300,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { exit(1); } - echo 'Currently at sequence ' . $aLastState['sequence_id'] . ' (' . $aLastState['lastimportdate'] . ') - ' . $aLastState['indexed'] . " indexed\n"; + echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n"; $sBatchEnd = $aLastState['lastimportdate']; $iEndSequence = $aLastState['sequence_id']; @@ -330,23 +323,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { if (file_exists($sImportFile)) { unlink($sImportFile); } - exec($sCMDDownload . ' -I ' . $iNextSeq, $aOutput, $iResult); + exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult); if ($iResult == 3) { - echo 'No new updates. Sleeping for ' . CONST_Replication_Recheck_Interval . " sec.\n"; + echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n"; sleep(CONST_Replication_Recheck_Interval); } elseif ($iResult != 0) { echo 'ERROR: updates failed.'; exit($iResult); } else { - $iEndSequence = (int) $aOutput[0]; + $iEndSequence = (int)$aOutput[0]; } } while ($iResult); // get the newest object from the diff file $sBatchEnd = 0; $iRet = 0; - exec(CONST_BasePath . '/utils/osm_file_date.py ' . $sImportFile, $sBatchEnd, $iRet); + exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet); if ($iRet == 5) { echo "Diff file is empty. skipping import.\n"; if (!$aResult['import-osmosis-all']) { @@ -362,7 +355,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { // Import the file $fCMDStartTime = time(); - echo $sCMDImport . "\n"; + echo $sCMDImport."\n"; unset($sJunk); $iErrorLevel = runWithEnv($sCMDImport, $aProcEnv); if ($iErrorLevel) { @@ -375,8 +368,8 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = 'INSERT INTO import_osmosis_log'; $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)'; $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'"; - $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','"; - $sSQL .= date('Y-m-d H:i:s') . "','import')"; + $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','"; + $sSQL .= date('Y-m-d H:i:s')."','import')"; var_Dump($sSQL); chksql($oDB->query($sSQL)); @@ -384,7 +377,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence"; var_Dump($sSQL); chksql($oDB->query($sSQL)); - echo date('Y-m-d H:i:s') . " Completed download step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n"; + echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; } // Index file @@ -402,20 +395,18 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { $sSQL = 'INSERT INTO import_osmosis_log'; $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)'; $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'"; - $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','"; - $sSQL .= date('Y-m-d H:i:s') . "','index')"; + $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','"; + $sSQL .= date('Y-m-d H:i:s')."','index')"; var_Dump($sSQL); $oDB->query($sSQL); - echo date('Y-m-d H:i:s') . " Completed index step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n"; + echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; $sSQL = 'update import_status set indexed = true'; $oDB->query($sSQL); } $fDuration = time() - $fStartTime; - echo date('Y-m-d H:i:s') . " Completed all for $sBatchEnd in " . round($fDuration / 60, 2) . " minutes\n"; - if (!$aResult['import-osmosis-all']) { - exit(0); - } + echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n"; + if (!$aResult['import-osmosis-all']) exit(0); } } From 9e35e5c2b02887e361972ffbb20e65bbba0d02c7 Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Sat, 8 Sep 2018 09:26:23 +0200 Subject: [PATCH 08/12] move checkModilePresence to class, delete own debug echo --- lib/classes/SetupClass.php | 11 ++++++++++- utils/update.php | 4 ---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/classes/SetupClass.php b/lib/classes/SetupClass.php index 120f625e..09f37e51 100755 --- a/lib/classes/SetupClass.php +++ b/lib/classes/SetupClass.php @@ -136,6 +136,11 @@ class SetupFunctions exit(1); } + // Try accessing the C module, so we know early if something is wrong + if (!checkModulePresence()) { + fail('error loading nominatim.so module'); + } + if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { echo 'Error: you need to download the country_osm_grid first:'; echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; @@ -219,6 +224,11 @@ class SetupFunctions { info('Create Functions'); + // Try accessing the C module, so we know eif something is wrong + // update.php calls this function + if (!checkModulePresence()) { + fail('error loading nominatim.so module'); + } $this->createSqlFunctions(); } @@ -330,7 +340,6 @@ class SetupFunctions } else { warn('wikipedia redirect dump file not found - some place importance values may be missing'); } - echo ' finish wikipedia'; } public function loadData($bDisableTokenPrecalc) diff --git a/utils/update.php b/utils/update.php index 3f9bf93a..0e12bb86 100755 --- a/utils/update.php +++ b/utils/update.php @@ -71,10 +71,6 @@ if ($aResult['init-updates']) { } if (!$aResult['no-update-functions']) { - // Try accessing the C module, - if (!checkModulePresence()) { - fail('error loading nominatim.so module'); - } // instantiate setupClass to use the function therein $cSetup = new SetupFunctions($aResult); $cSetup->createFunctions(); From 08c2f03ccc2a4a31d0d30dd4260479ac647d884e Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Sat, 8 Sep 2018 10:14:08 +0200 Subject: [PATCH 09/12] moving comment to right position --- utils/setup.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/setup.php b/utils/setup.php index 63c4d211..96d66a16 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -3,9 +3,10 @@ require_once(dirname(dirname(__FILE__)).'/settings/settings.php'); require_once(CONST_BasePath.'/lib/init-cmd.php'); -require_once(CONST_BasePath.'/lib/classes/SetupClass.php'); -// ->indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt +// ->indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt // ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB() + +require_once(CONST_BasePath.'/lib/classes/SetupClass.php'); require_once(CONST_BasePath.'/lib/setup_functions.php'); ini_set('memory_limit', '800M'); From a0dbeabed1a18e9bedb6cb2a6eed4c2b5fbf45cc Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Mon, 17 Sep 2018 10:28:00 +0200 Subject: [PATCH 10/12] move setupclass, move command line array, remove args from update array --- lib/{classes => setup}/SetupClass.php | 15 +++-- lib/setup_functions.php | 85 --------------------------- utils/setup.php | 40 ++++++++++++- utils/update.php | 40 ++++++++++++- 4 files changed, 86 insertions(+), 94 deletions(-) rename lib/{classes => setup}/SetupClass.php (98%) diff --git a/lib/classes/SetupClass.php b/lib/setup/SetupClass.php similarity index 98% rename from lib/classes/SetupClass.php rename to lib/setup/SetupClass.php index 09f37e51..f31c2353 100755 --- a/lib/classes/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -16,7 +16,7 @@ class SetupFunctions protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData // pgsqlRunPartitionScript, calculatePostcodes, ..if no already set - public function __construct($aCMDResult) + public function __construct($aCMDResult, $callingFunction = 'setup') { // by default, use all but one processor, but never more than 15. $this->iInstances = isset($aCMDResult['threads']) @@ -45,7 +45,6 @@ class SetupFunctions // setting member variables based on command line options stored in $aCMDResult $this->sVerbose = $aCMDResult['verbose']; - $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; //setting default values which are not set by the update.php array if (isset($aCMDResult['ignore-errors'])) { @@ -63,6 +62,14 @@ class SetupFunctions } else { $this->bNoPartitions = false; } + + // if class is instantiated by update.php, we have to set EnableDiffUpdates to true + // otherwise set to value provided comand line to setup.php + if ($callingFunction == 'update') { + $this->bEnableDiffUpdates = true; + } elseif ($callingFunction == 'setup') { + $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; + } } public function createDB() @@ -166,8 +173,8 @@ class SetupFunctions $this->pgsqlRunScript('update country_name set partition = 0'); } - // the following will be needed by create_functions later but - // is only defined in the subsequently called T + // the following will be needed by createFunctions later but + // is only defined in the subsequently called createTables // Create dummies here that will be overwritten by the proper // versions in create-tables. $this->pgsqlRunScript('CREATE TABLE IF NOT EXISTS place_boundingbox ()'); diff --git a/lib/setup_functions.php b/lib/setup_functions.php index af718ed0..b1417678 100755 --- a/lib/setup_functions.php +++ b/lib/setup_functions.php @@ -34,90 +34,5 @@ function checkModulePresence() echo $oResult->userinfo . "\n\n"; $bResult = false; } - return $bResult; } - -// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) -// create and array -function createSetupArgvArray() -{ - $aCMDOptions - = array( - 'Create and setup nominatim search system', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - - array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), - array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), - - array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), - - array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), - array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), - array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), - array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), - array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), - array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), - array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), - array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), - array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), - array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), - array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), - array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), - array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), - array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), - array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), - array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), - array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), - array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), - array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), - array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), - ); - return $aCMDOptions; -} - -function createUpdateArgvArray() -{ - $aCMDOptions - = array( - 'Import / update / index osm data', - array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), - array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), - array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), - - array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), - array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), - array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'), - array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), - array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), - array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), - - array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'), - - array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'), - array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'), - array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), - - array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'), - array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'), - array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'), - array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'), - - array('index', '', 0, 1, 0, 0, 'bool', 'Index'), - array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'), - array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'), - - array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'), - array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'), - array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'), - - array('create-functions', '', 0, 1, 1, 1, 'bool', 'Create functions'), - array('enable-diff-updates', '', 0, 1, 1, 1, 'bool', 'Turn on the code required to make diff updates work'), - ); - return $aCMDOptions; -} diff --git a/utils/setup.php b/utils/setup.php index 96d66a16..5bc2a6f9 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -6,13 +6,49 @@ require_once(CONST_BasePath.'/lib/init-cmd.php'); // ->indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt // ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB() -require_once(CONST_BasePath.'/lib/classes/SetupClass.php'); +require_once(CONST_BasePath.'/lib/setup/SetupClass.php'); require_once(CONST_BasePath.'/lib/setup_functions.php'); ini_set('memory_limit', '800M'); use Nominatim\Setup\SetupFunctions as SetupFunctions; -$aCMDOptions = createSetupArgvArray(); +// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) +$aCMDOptions += array( + 'Create and setup nominatim search system', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), + array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), + + array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), + + array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), + array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), + array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), + array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), + array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), + array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), + array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), + array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), + array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), + array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), + array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), + array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), + array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), + array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), + array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), + array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), + array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), + array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), + array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), + array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), +); // $aCMDOptions passed to getCmdOpt by reference getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); diff --git a/utils/update.php b/utils/update.php index 0e12bb86..9175a617 100755 --- a/utils/update.php +++ b/utils/update.php @@ -4,13 +4,47 @@ require_once(dirname(dirname(__FILE__)).'/settings/settings.php'); require_once(CONST_BasePath.'/lib/init-cmd.php'); require_once(CONST_BasePath.'/lib/setup_functions.php'); -require_once(CONST_BasePath.'/lib/classes/SetupClass.php'); +require_once(CONST_BasePath.'/lib/setup/SetupClass.php'); ini_set('memory_limit', '800M'); use Nominatim\Setup\SetupFunctions as SetupFunctions; -$aCMDOptions = createUpdateArgvArray(); +// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) +$aCMDOptions += array( + 'Import / update / index osm data', + array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), + array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), + array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), + + array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'), + array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'), + array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'), + array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'), + array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'), + array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'), + + array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'), + + array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'), + array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'), + array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), + + array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'), + array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'), + array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'), + array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'), + + array('index', '', 0, 1, 0, 0, 'bool', 'Index'), + array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'), + array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'), + + array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'), + array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'), + array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'), +); + getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true); if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1; @@ -72,7 +106,7 @@ if ($aResult['init-updates']) { if (!$aResult['no-update-functions']) { // instantiate setupClass to use the function therein - $cSetup = new SetupFunctions($aResult); + $cSetup = new SetupFunctions($aResult, 'update'); $cSetup->createFunctions(); } From a948050015060896edcdd9de4f9a304b1569a732 Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Tue, 18 Sep 2018 09:17:54 +0200 Subject: [PATCH 11/12] typo --- lib/setup/SetupClass.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php index f31c2353..927aac5f 100755 --- a/lib/setup/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -64,7 +64,7 @@ class SetupFunctions } // if class is instantiated by update.php, we have to set EnableDiffUpdates to true - // otherwise set to value provided comand line to setup.php + // otherwise set to value provided by setup.php's command line arg array if ($callingFunction == 'update') { $this->bEnableDiffUpdates = true; } elseif ($callingFunction == 'setup') { From 0273e128f432c0a683d9c68cfd7d4bd29793c925 Mon Sep 17 00:00:00 2001 From: ThomasBarris Date: Tue, 18 Sep 2018 21:28:05 +0200 Subject: [PATCH 12/12] change variables for class SetupClass.php instantiation --- lib/setup/SetupClass.php | 2 +- utils/setup.php | 2 +- utils/update.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php index f31c2353..3b4f683d 100755 --- a/lib/setup/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -16,7 +16,7 @@ class SetupFunctions protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData // pgsqlRunPartitionScript, calculatePostcodes, ..if no already set - public function __construct($aCMDResult, $callingFunction = 'setup') + public function __construct($callingFunction, array $aCMDResult = array()) { // by default, use all but one processor, but never more than 15. $this->iInstances = isset($aCMDResult['threads']) diff --git a/utils/setup.php b/utils/setup.php index 5bc2a6f9..bdd3f03b 100755 --- a/utils/setup.php +++ b/utils/setup.php @@ -71,7 +71,7 @@ if ($aCMDResult['osmosis-init']) { // ****************************************************** // instantiate Setup class -$cSetup = new SetupFunctions($aCMDResult); +$cSetup = new SetupFunctions('setup', $aCMDResult); // ******************************************************* // go through complete process if 'all' is selected or start selected functions diff --git a/utils/update.php b/utils/update.php index 9175a617..dc8389ab 100755 --- a/utils/update.php +++ b/utils/update.php @@ -106,7 +106,7 @@ if ($aResult['init-updates']) { if (!$aResult['no-update-functions']) { // instantiate setupClass to use the function therein - $cSetup = new SetupFunctions($aResult, 'update'); + $cSetup = new SetupFunctions('update'); $cSetup->createFunctions(); }