Merge pull request #2181 from lonvia/port-more-tool-functions-to-python

Port more tool functions to python

lib-php/DB.php

@@ -240,16 +240,6 @@ class DB
         return ($this->getOne($sSQL, array(':tablename' => $sTableName)) == 1);
     }
 
-    /**
-    * Returns a list of table names in the database
-    *
-    * @return array[]
-    */
-    public function getListOfTables()
-    {
-        return $this->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'");
-    }
-
     /**
      * Deletes a table. Returns true if deleted or didn't exist.
      *
@@ -262,76 +252,6 @@ class DB
         return $this->exec('DROP TABLE IF EXISTS '.$sTableName.' CASCADE') == 0;
     }
 
-    /**
-    * Check if an index exists in the database. Optional filtered by tablename
-    *
-    * @param string  $sTableName
-    *
-    * @return boolean
-    */
-    public function indexExists($sIndexName, $sTableName = null)
-    {
-        return in_array($sIndexName, $this->getListOfIndices($sTableName));
-    }
-
-    /**
-    * Returns a list of index names in the database, optional filtered by tablename
-    *
-    * @param string  $sTableName
-    *
-    * @return array
-    */
-    public function getListOfIndices($sTableName = null)
-    {
-        //  table_name            | index_name                      | column_name
-        // -----------------------+---------------------------------+--------------
-        //  country_name          | idx_country_name_country_code   | country_code
-        //  country_osm_grid      | idx_country_osm_grid_geometry   | geometry
-        //  import_polygon_delete | idx_import_polygon_delete_osmid | osm_id
-        //  import_polygon_delete | idx_import_polygon_delete_osmid | osm_type
-        //  import_polygon_error  | idx_import_polygon_error_osmid  | osm_id
-        //  import_polygon_error  | idx_import_polygon_error_osmid  | osm_type
-        $sSql = <<< END
-SELECT
-    t.relname as table_name,
-    i.relname as index_name,
-    a.attname as column_name
-FROM
-    pg_class t,
-    pg_class i,
-    pg_index ix,
-    pg_attribute a
-WHERE
-    t.oid = ix.indrelid
-    and i.oid = ix.indexrelid
-    and a.attrelid = t.oid
-    and a.attnum = ANY(ix.indkey)
-    and t.relkind = 'r'
-    and i.relname NOT LIKE 'pg_%'
-    FILTERS
- ORDER BY
-    t.relname,
-    i.relname,
-    a.attname
-END;
-
-        $aRows = null;
-        if ($sTableName) {
-            $sSql = str_replace('FILTERS', 'and t.relname = :tablename', $sSql);
-            $aRows = $this->getAll($sSql, array(':tablename' => $sTableName));
-        } else {
-            $sSql = str_replace('FILTERS', '', $sSql);
-            $aRows = $this->getAll($sSql);
-        }
-
-        $aIndexNames = array_unique(array_map(function ($aRow) {
-            return $aRow['index_name'];
-        }, $aRows));
-        sort($aIndexNames);
-
-        return $aIndexNames;
-    }
-
     /**
      * Tries to connect to the database but on failure doesn't throw an exception.
      *

lib-php/admin/check_import_finished.php

@@ -5,197 +5,6 @@ require_once(CONST_LibDir.'/init-cmd.php');
 
 loadSettings(getcwd());
 
-$term_colors = array(
-                'green' => "\033[92m",
-                'red' => "\x1B[31m",
-                'normal' => "\033[0m"
-);
-
-$print_success = function ($message = 'OK') use ($term_colors) {
-    echo $term_colors['green'].$message.$term_colors['normal']."\n";
-};
-
-$print_fail = function ($message = 'Failed') use ($term_colors) {
-    echo $term_colors['red'].$message.$term_colors['normal']."\n";
-};
-
-$oDB = new Nominatim\DB;
-
-
-function isReverseOnlyInstallation()
-{
-    global $oDB;
-    return !$oDB->tableExists('search_name');
-}
-
-// Check (guess) if the setup.php included --drop
-function isNoUpdateInstallation()
-{
-    global $oDB;
-    return $oDB->tableExists('placex') && !$oDB->tableExists('planet_osm_rels') ;
-}
-
-
-echo 'Checking database got created ... ';
-if ($oDB->checkConnection()) {
-    $print_success();
-} else {
-    $print_fail();
-    echo <<< END
-    Hints:
-    * Is the database server started?
-    * Check the NOMINATIM_DATABASE_DSN variable in your local .env
-    * Try connecting to the database with the same settings
-
-END;
-    exit(1);
-}
-
-
-echo 'Checking nominatim.so module installed ... ';
-$sStandardWord = $oDB->getOne("SELECT make_standard_name('a')");
-if ($sStandardWord === 'a') {
-    $print_success();
-} else {
-    $print_fail();
-    echo <<< END
-    The Postgresql extension nominatim.so was not found in the database.
-    Hints:
-    * Check the output of the CMmake/make installation step
-    * Does nominatim.so exist?
-    * Does nominatim.so exist on the database server?
-    * Can nominatim.so be accessed by the database user?
-
-END;
-    exit(1);
-}
-
-if (!isNoUpdateInstallation()) {
-    echo 'Checking place table ... ';
-    if ($oDB->tableExists('place')) {
-        $print_success();
-    } else {
-        $print_fail();
-        echo <<< END
-        * The import didn't finish.
-        Hints:
-        * Check the output of the utils/setup.php you ran.
-        Usually the osm2pgsql step failed. Check for errors related to
-        * the file you imported not containing any places
-        * harddrive full
-        * out of memory (RAM)
-        * osm2pgsql killed by other scripts, for consuming to much memory
-
-    END;
-        exit(1);
-    }
-}
-
-
-echo 'Checking indexing status ... ';
-$iUnindexed = $oDB->getOne('SELECT count(*) FROM placex WHERE indexed_status > 0');
-if ($iUnindexed == 0) {
-    $print_success();
-} else {
-    $print_fail();
-    echo <<< END
-    The indexing didn't finish. There is still $iUnindexed places. See the
-    question 'Can a stopped/killed import process be resumed?' in the
-    troubleshooting guide.
-
-END;
-    exit(1);
-}
-
-echo "Search index creation\n";
-$aExpectedIndices = array(
-    // sql/indices.src.sql
-    'idx_word_word_id',
-    'idx_place_addressline_address_place_id',
-    'idx_placex_rank_search',
-    'idx_placex_rank_address',
-    'idx_placex_parent_place_id',
-    'idx_placex_geometry_reverse_lookuppolygon',
-    'idx_placex_geometry_reverse_placenode',
-    'idx_osmline_parent_place_id',
-    'idx_osmline_parent_osm_id',
-    'idx_postcode_id',
-    'idx_postcode_postcode'
-);
-if (!isReverseOnlyInstallation()) {
-    $aExpectedIndices = array_merge($aExpectedIndices, array(
-        // sql/indices_search.src.sql
-        'idx_search_name_nameaddress_vector',
-        'idx_search_name_name_vector',
-        'idx_search_name_centroid'
-    ));
-}
-if (!isNoUpdateInstallation()) {
-    $aExpectedIndices = array_merge($aExpectedIndices, array(
-        'idx_placex_pendingsector',
-        'idx_location_area_country_place_id',
-        'idx_place_osm_unique',
-    ));
-}
-
-foreach ($aExpectedIndices as $sExpectedIndex) {
-    echo "Checking index $sExpectedIndex ... ";
-    if ($oDB->indexExists($sExpectedIndex)) {
-        $print_success();
-    } else {
-        $print_fail();
-        echo <<< END
-        Hints:
-        * Run './utils/setup.php --create-search-indices --ignore-errors' to
-          create missing indices.
-
-END;
-        exit(1);
-    }
-}
-
-echo 'Checking search indices are valid ... ';
-$sSQL = <<< END
-    SELECT relname
-    FROM pg_class, pg_index
-    WHERE pg_index.indisvalid = false
-      AND pg_index.indexrelid = pg_class.oid;
-END;
-$aInvalid = $oDB->getCol($sSQL);
-if (empty($aInvalid)) {
-    $print_success();
-} else {
-    $print_fail();
-    echo <<< END
-    At least one index is invalid. That can happen, e.g. when index creation was
-    disrupted and later restarted. You should delete the affected indices and
-    run the index stage of setup again.
-    See the question 'Can a stopped/killed import process be resumed?' in the
-    troubleshooting guide.
-    Affected indices: 
-END;
-    echo join(', ', $aInvalid) . "\n";
-    exit(1);
-}
-
-
-
-if (getSettingBool('USE_US_TIGER_DATA')) {
-    echo 'Checking TIGER table exists ... ';
-    if ($oDB->tableExists('location_property_tiger')) {
-        $print_success();
-    } else {
-        $print_fail();
-        echo <<< END
-        Table 'location_property_tiger' does not exist. Run the TIGER data
-        import again.
-
-END;
-        exit(1);
-    }
-}
-
-
-
-
-exit(0);
+(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL')))
+    ->addParams('admin', '--check-database')
+    ->run();

lib-php/lib.php

@@ -132,24 +132,6 @@ function addQuotes($s)
     return "'".$s."'";
 }
 
-function fwriteConstDef($rFile, $sConstName, $value)
-{
-    $sEscapedValue;
-
-    if (is_bool($value)) {
-        $sEscapedValue = $value ? 'true' : 'false';
-    } elseif (is_numeric($value)) {
-        $sEscapedValue = strval($value);
-    } elseif (!$value) {
-        $sEscapedValue = 'false';
-    } else {
-        $sEscapedValue = addQuotes(str_replace("'", "\\'", (string)$value));
-    }
-
-    fwrite($rFile, "@define('CONST_$sConstName', $sEscapedValue);\n");
-}
-
-
 function parseLatLon($sQuery)
 {
     $sFound    = null;
@@ -226,17 +208,6 @@ function parseLatLon($sQuery)
     return array($sFound, $fQueryLat, $fQueryLon);
 }
 
-function createPointsAroundCenter($fLon, $fLat, $fRadius)
-{
-    $iSteps = max(8, min(100, ($fRadius * 40000)^2));
-    $fStepSize = (2*pi())/$iSteps;
-    $aPolyPoints = array();
-    for ($f = 0; $f < 2*pi(); $f += $fStepSize) {
-        $aPolyPoints[] = array('', $fLon+($fRadius*sin($f)), $fLat+($fRadius*cos($f)) );
-    }
-    return $aPolyPoints;
-}
-
 function closestHouseNumber($aRow)
 {
     $fHouse = $aRow['startnumber']
@@ -256,25 +227,3 @@ function closestHouseNumber($aRow)
 
     return max(min($aRow['endnumber'], $iHn), $aRow['startnumber']);
 }
-
-function getSearchRankLabel($iRank)
-{
-    if (!isset($iRank)) return 'unknown';
-    if ($iRank < 2) return 'continent';
-    if ($iRank < 4) return 'sea';
-    if ($iRank < 8) return 'country';
-    if ($iRank < 12) return 'state';
-    if ($iRank < 16) return 'county';
-    if ($iRank == 16) return 'city';
-    if ($iRank == 17) return 'town / island';
-    if ($iRank == 18) return 'village / hamlet';
-    if ($iRank == 20) return 'suburb';
-    if ($iRank == 21) return 'postcode area';
-    if ($iRank == 22) return 'croft / farm / locality / islet';
-    if ($iRank == 23) return 'postcode area';
-    if ($iRank == 25) return 'postcode point';
-    if ($iRank == 26) return 'street / major landmark';
-    if ($iRank == 27) return 'minory street / path';
-    if ($iRank == 28) return 'house / building';
-    return 'other: ' . $iRank;
-}

lib-php/setup/SetupClass.php

@@ -657,50 +657,7 @@ class SetupFunctions
 
     public function drop()
     {
-        info('Drop tables only required for updates');
-
-        // The implementation is potentially a bit dangerous because it uses
-        // a positive selection of tables to keep, and deletes everything else.
-        // Including any tables that the unsuspecting user might have manually
-        // created. USE AT YOUR OWN PERIL.
-        // tables we want to keep. everything else goes.
-        $aKeepTables = array(
-                        '*columns',
-                        'import_polygon_*',
-                        'import_status',
-                        'place_addressline',
-                        'location_postcode',
-                        'location_property*',
-                        'placex',
-                        'search_name',
-                        'seq_*',
-                        'word',
-                        'query_log',
-                        'new_query_log',
-                        'spatial_ref_sys',
-                        'country_name',
-                        'place_classtype_*',
-                        'country_osm_grid'
-                       );
-
-        $aDropTables = array();
-        $aHaveTables = $this->db()->getListOfTables();
-
-        foreach ($aHaveTables as $sTable) {
-            $bFound = false;
-            foreach ($aKeepTables as $sKeep) {
-                if (fnmatch($sKeep, $sTable)) {
-                    $bFound = true;
-                    break;
-                }
-            }
-            if (!$bFound) array_push($aDropTables, $sTable);
-        }
-        foreach ($aDropTables as $sDrop) {
-            $this->dropTable($sDrop);
-        }
-
-        $this->removeFlatnodeFile();
+        (clone($this->oNominatimCmd))->addParams('freeze')->run();
     }
 
     /**
@@ -710,48 +667,7 @@ class SetupFunctions
      */
     public function setupWebsite()
     {
-        if (!is_dir(CONST_InstallDir.'/website')) {
-            info('Creating directory for website scripts at: '.CONST_InstallDir.'/website');
-            mkdir(CONST_InstallDir.'/website');
-        }
-
-        $aScripts = array(
-          'deletable.php',
-          'details.php',
-          'lookup.php',
-          'polygons.php',
-          'reverse.php',
-          'search.php',
-          'status.php'
-        );
-
-        foreach ($aScripts as $sScript) {
-            $rFile = fopen(CONST_InstallDir.'/website/'.$sScript, 'w');
-
-            fwrite($rFile, "<?php\n\n");
-            fwrite($rFile, '@define(\'CONST_Debug\', $_GET[\'debug\'] ?? false);'."\n\n");
-
-            fwriteConstDef($rFile, 'LibDir', CONST_LibDir);
-            fwriteConstDef($rFile, 'Database_DSN', getSetting('DATABASE_DSN'));
-            fwriteConstDef($rFile, 'Default_Language', getSetting('DEFAULT_LANGUAGE'));
-            fwriteConstDef($rFile, 'Log_DB', getSettingBool('LOG_DB'));
-            fwriteConstDef($rFile, 'Log_File', getSetting('LOG_FILE'));
-            fwriteConstDef($rFile, 'Max_Word_Frequency', (int)getSetting('MAX_WORD_FREQUENCY'));
-            fwriteConstDef($rFile, 'NoAccessControl', getSettingBool('CORS_NOACCESSCONTROL'));
-            fwriteConstDef($rFile, 'Places_Max_ID_count', (int)getSetting('LOOKUP_MAX_COUNT'));
-            fwriteConstDef($rFile, 'PolygonOutput_MaximumTypes', getSetting('POLYGON_OUTPUT_MAX_TYPES'));
-            fwriteConstDef($rFile, 'Search_BatchMode', getSettingBool('SEARCH_BATCH_MODE'));
-            fwriteConstDef($rFile, 'Search_NameOnlySearchFrequencyThreshold', getSetting('SEARCH_NAME_ONLY_THRESHOLD'));
-            fwriteConstDef($rFile, 'Term_Normalization_Rules', getSetting('TERM_NORMALIZATION'));
-            fwriteConstDef($rFile, 'Use_Aux_Location_data', getSettingBool('USE_AUX_LOCATION_DATA'));
-            fwriteConstDef($rFile, 'Use_US_Tiger_Data', getSettingBool('USE_US_TIGER_DATA'));
-            fwriteConstDef($rFile, 'MapIcon_URL', getSetting('MAPICON_URL'));
-
-            fwrite($rFile, 'require_once(\''.CONST_LibDir.'/website/'.$sScript."');\n");
-            fclose($rFile);
-
-            chmod(CONST_InstallDir.'/website/'.$sScript, 0755);
-        }
+        (clone($this->oNominatimCmd))->addParams('refresh', '--website')->run();
     }
 
     /**

lib-php/website/details.php

@@ -53,7 +53,7 @@ if ($sOsmType && $iOsmId > 0) {
 
     // Be nice about our error messages for broken geometry
 
-    if (!$sPlaceId) {
+    if (!$sPlaceId && $oDB->tableExists('import_polygon_error')) {
         $sSQL = 'SELECT ';
         $sSQL .= '    osm_type, ';
         $sSQL .= '    osm_id, ';
@@ -144,7 +144,6 @@ if (!$aPointDetails) {
 }
 
 $aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber'];
-$aPointDetails['rank_search_label'] = getSearchRankLabel($aPointDetails['rank_search']); // only used in HTML format
 
 // Get all alternative names (languages, etc)
 $sSQL = 'SELECT (each(name)).key,(each(name)).value FROM placex ';