Added fixture for sql_preprocessor and fixed some issues

nominatim/tools/tiger_data.py

@@ -44,9 +44,7 @@ def handle_threaded_sql_statements(sel, file):
     lines = 0
     end_of_file = False
     # Using pool of database connections to execute sql statements
-    while True:
+    while not end_of_file:
-        if end_of_file:
-            break
         for key, _ in sel.select(1):
             conn = key.data
             try:
@@ -61,7 +59,7 @@ def handle_threaded_sql_statements(sel, file):
                         print('. ', end='', flush=True)
                         lines = 0
             except Exception as exc: # pylint: disable=broad-except
-                LOG.error('Wrong SQL statement: %s', exc)
+                LOG.info('Wrong SQL statement: %s', exc)
 
 
 def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):

test/python/conftest.py

@@ -13,6 +13,7 @@ sys.path.insert(0, str(SRC_DIR.resolve()))
 
 from nominatim.config import Configuration
 from nominatim.db import connection
+from nominatim.db.sql_preprocessor import SQLPreprocessor
 
 class _TestingCursor(psycopg2.extras.DictCursor):
     """ Extension to the DictCursor class that provides execution
@@ -266,3 +267,9 @@ def osm2pgsql_options(temp_db):
                 flatnode_file='',
                 tablespaces=dict(slim_data='', slim_index='',
                                  main_data='', main_index=''))
+
+@pytest.fixture
+def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory):
+    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
+    table_factory('country_name', 'partition INT', (0, 1, 2))
+    return SQLPreprocessor(temp_db_conn, def_config, tmp_path)

test/python/test_db_sql_preprocessor.py

@@ -5,8 +5,6 @@ from pathlib import Path
 
 import pytest
 
-from nominatim.db.sql_preprocessor import SQLPreprocessor
-
 @pytest.fixture
 def sql_factory(tmp_path):
     def _mk_sql(sql_body):
@@ -21,13 +19,6 @@ def sql_factory(tmp_path):
 
     return _mk_sql
 
-
-@pytest.fixture
-def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory):
-    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
-    table_factory('country_name', 'partition INT', (0, 1, 2))
-    return SQLPreprocessor(temp_db_conn, def_config, tmp_path)
-
 @pytest.mark.parametrize("expr,ret", [
     ("'a'", 'a'),
     ("'{{db.partitions|join}}'", '012'),

test/python/test_tools_tiger_data.py

@@ -10,15 +10,11 @@ from nominatim.tools import tiger_data, database_import
 
 
 @pytest.mark.parametrize("threads", (1, 5))
-def test_add_tiger_data(dsn, src_dir, def_config, monkeypatch,tmp_path,
+def test_add_tiger_data(dsn, src_dir, def_config, tmp_path, sql_preprocessor,
                         temp_db_cursor, threads, temp_db):
-    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
     temp_db_cursor.execute('CREATE EXTENSION hstore')
     temp_db_cursor.execute('CREATE EXTENSION postgis')
     temp_db_cursor.execute('CREATE TABLE place (id INT)')
-
-    database_import.import_base_data('dbname=' + temp_db, src_dir / 'data',
-                                     ignore_partitions=False)
     sqlfile = tmp_path / '1010.sql'
     sqlfile.write_text("""INSERT INTO place values (1)""")
     tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql')
@@ -26,15 +22,11 @@ def test_add_tiger_data(dsn, src_dir, def_config, monkeypatch,tmp_path,
     assert temp_db_cursor.table_rows('place') == 1
 
 @pytest.mark.parametrize("threads", (1, 5))
-def test_add_tiger_data_tarfile(dsn, src_dir, def_config, monkeypatch,tmp_path,
+def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path,
-                        temp_db_cursor, threads, temp_db):
+                        temp_db_cursor, threads, temp_db, sql_preprocessor):
-    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
     temp_db_cursor.execute('CREATE EXTENSION hstore')
     temp_db_cursor.execute('CREATE EXTENSION postgis')
     temp_db_cursor.execute('CREATE TABLE place (id INT)')
-
-    database_import.import_base_data('dbname=' + temp_db, src_dir / 'data',
-                                     ignore_partitions=False)
     sqlfile = tmp_path / '1010.sql'
     sqlfile.write_text("""INSERT INTO place values (1)""")
     tar = tarfile.open("sample.tar.gz", "w:gz")