move import-data option to native python

This adds a new dependecy to the Python psutil package.

test/python/conftest.py

@@ -198,4 +198,13 @@ def placex_table(temp_db_with_extensions, temp_db_conn):
     temp_db_conn.commit()
 
 
-
+@pytest.fixture
+def osm2pgsql_options(temp_db):
+    return dict(osm2pgsql='echo',
+                osm2pgsql_cache=10,
+                osm2pgsql_style='style.file',
+                threads=1,
+                dsn='dbname=' + temp_db,
+                flatnode_file='',
+                tablespaces=dict(slim_data='', slim_index='',
+                                 main_data='', main_index=''))

test/python/test_cli.py

@@ -40,6 +40,7 @@ def mock_run_legacy(monkeypatch):
     monkeypatch.setattr(nominatim.cli, 'run_legacy_script', mock)
     return mock
 
+
 @pytest.fixture
 def mock_func_factory(monkeypatch):
     def get_mock(module, func):
@@ -49,6 +50,7 @@ def mock_func_factory(monkeypatch):
 
     return get_mock
 
+
 def test_cli_help(capsys):
     """ Running nominatim tool without arguments prints help.
     """

test/python/test_db_connection.py

@@ -2,6 +2,7 @@
 Tests for specialised conenction and cursor classes.
 """
 import pytest
+import psycopg2
 
 from nominatim.db.connection import connect, get_pg_env
 
@@ -30,6 +31,22 @@ def test_connection_index_exists(db, temp_db_cursor):
     assert db.index_exists('some_index', table='bar') == False
 
 
+def test_drop_table_existing(db, temp_db_cursor):
+    temp_db_cursor.execute('CREATE TABLE dummy (id INT)')
+
+    assert db.table_exists('dummy')
+    db.drop_table('dummy')
+    assert not db.table_exists('dummy')
+
+
+def test_drop_table_non_existsing(db):
+    db.drop_table('dfkjgjriogjigjgjrdghehtre')
+
+
+def test_drop_table_non_existing_force(db):
+    with pytest.raises(psycopg2.ProgrammingError, match='.*does not exist.*'):
+        db.drop_table('dfkjgjriogjigjgjrdghehtre', if_exists=False)
+
 def test_connection_server_version_tuple(db):
     ver = db.server_version_tuple()
 

test/python/test_tools_database_import.py

@@ -4,6 +4,7 @@ Tests for functions to import a new database.
 import pytest
 import psycopg2
 import sys
+from pathlib import Path
 
 from nominatim.tools import database_import
 from nominatim.errors import UsageError
@@ -94,3 +95,42 @@ def test_import_base_data_ignore_partitions(src_dir, temp_db, temp_db_cursor):
 
     assert temp_db_cursor.scalar('SELECT count(*) FROM country_name') > 0
     assert temp_db_cursor.scalar('SELECT count(*) FROM country_name WHERE partition != 0') == 0
+
+
+def test_import_osm_data_simple(temp_db_cursor,osm2pgsql_options):
+    temp_db_cursor.execute('CREATE TABLE place (id INT)')
+    temp_db_cursor.execute('INSERT INTO place values (1)')
+
+    database_import.import_osm_data('file.pdf', osm2pgsql_options)
+
+
+def test_import_osm_data_simple_no_data(temp_db_cursor,osm2pgsql_options):
+    temp_db_cursor.execute('CREATE TABLE place (id INT)')
+
+    with pytest.raises(UsageError, match='No data.*'):
+        database_import.import_osm_data('file.pdf', osm2pgsql_options)
+
+
+def test_import_osm_data_drop(temp_db_conn, temp_db_cursor, tmp_path, osm2pgsql_options):
+    temp_db_cursor.execute('CREATE TABLE place (id INT)')
+    temp_db_cursor.execute('CREATE TABLE planet_osm_nodes (id INT)')
+    temp_db_cursor.execute('INSERT INTO place values (1)')
+
+    flatfile = tmp_path / 'flatfile'
+    flatfile.write_text('touch')
+
+    osm2pgsql_options['flatnode_file'] = str(flatfile.resolve())
+
+    database_import.import_osm_data('file.pdf', osm2pgsql_options, drop=True)
+
+    assert not flatfile.exists()
+    assert not temp_db_conn.table_exists('planet_osm_nodes')
+
+
+def test_import_osm_data_default_cache(temp_db_cursor,osm2pgsql_options):
+    temp_db_cursor.execute('CREATE TABLE place (id INT)')
+    temp_db_cursor.execute('INSERT INTO place values (1)')
+
+    osm2pgsql_options['osm2pgsql_cache'] = 0
+
+    database_import.import_osm_data(Path(__file__), osm2pgsql_options)

test/python/test_tools_exec_utils.py

@@ -105,8 +105,15 @@ def test_run_api_with_extra_env(tmp_project_dir):
 
 ### run_osm2pgsql
 
-def test_run_osm2pgsql():
-    exec_utils.run_osm2pgsql(dict(osm2pgsql='echo', append=False, flatnode_file=None,
-                                  dsn='dbname=foobar', threads=1, osm2pgsql_cache=500,
-                                  osm2pgsql_style='./my.style',
-                                  import_file='foo.bar'))
+def test_run_osm2pgsql(osm2pgsql_options):
+    osm2pgsql_options['append'] = False
+    osm2pgsql_options['import_file'] = 'foo.bar'
+    osm2pgsql_options['tablespaces']['osm_data'] = 'extra'
+    exec_utils.run_osm2pgsql(osm2pgsql_options)
+
+
+def test_run_osm2pgsql_disable_jit(osm2pgsql_options):
+    osm2pgsql_options['append'] = True
+    osm2pgsql_options['import_file'] = 'foo.bar'
+    osm2pgsql_options['disable_jit'] = True
+    exec_utils.run_osm2pgsql(osm2pgsql_options)