hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-14 10:27:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

simplify interface for adding tiger data

Browse Source 
Also simplifies tests using existing fixtures.
This commit is contained in:
Sarah Hoffmann
2021-04-19 09:23:37 +02:00
parent 995ba2c7c2
commit 8f63f9516b
4 changed files with 31 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
nominatim/cli.py
View File

@@ -148,11 +148,8 @@ class UpdateAddData:
@staticmethod
def run(args):
if args.tiger_data:
return tiger_data.add_tiger_data(args.config.get_libpq_dsn(),
args.tiger_data,
args.threads or 1,
args.config,
args.sqllib_dir)
return tiger_data.add_tiger_data(args.tiger_data,
args.config, args.threads or 1)
params = ['update.php']
if args.file:

10
nominatim/tools/tiger_data.py
View File

@@ -76,17 +76,17 @@ def handle_unregister_connection_pool(sel, place_threads):
conn.close()
place_threads -= 1
def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
""" Import tiger data from directory or tar file
def add_tiger_data(data_dir, config, threads):
""" Import tiger data from directory or tar file `data dir`.
"""
dsn = config.get_libpq_dsn()
sql_files, tar = handle_tarfile_or_directory(data_dir)
if not sql_files:
return
with connect(dsn) as conn:
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql = SQLPreprocessor(conn, config, config.lib_dir.sql)
sql.run_sql_file(conn, 'tiger_import_start.sql')
# Reading sql_files and then for each file line handling
@@ -116,5 +116,5 @@ def add_tiger_data(dsn, data_dir, threads, config, sqllib_dir):
print('\n')
LOG.warning("Creating indexes on Tiger data")
with connect(dsn) as conn:
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql = SQLPreprocessor(conn, config, config.lib_dir.sql)
sql.run_sql_file(conn, 'tiger_import_finish.sql')

7
test/python/conftest.py
View File

@@ -126,7 +126,12 @@ def table_factory(temp_db_cursor):
@pytest.fixture
def def_config():
return Configuration(None, SRC_DIR.resolve() / 'settings')
cfg = Configuration(None, SRC_DIR.resolve() / 'settings')
cfg.set_libdirs(module='.', osm2pgsql='.',
php=SRC_DIR / 'lib-php',
sql=SRC_DIR / 'lib-sql',
data=SRC_DIR / 'data')
return cfg
@pytest.fixture
def src_dir():

41
test/python/test_tools_tiger_data.py
View File

@@ -10,35 +10,31 @@ from nominatim.tools import tiger_data, database_import
@pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data(dsn, src_dir, def_config, tmp_path, sql_preprocessor,
temp_db_cursor, threads, temp_db):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
def test_add_tiger_data(def_config, tmp_path, sql_preprocessor,
temp_db_cursor, threads, temp_db_with_extensions):
temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.sql'
sqlfile.write_text("""INSERT INTO place values (1);
INSERT INTO non_existant_table values (1);""")
tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql')
tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
assert temp_db_cursor.table_rows('place') == 1
@pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data_bad_file(dsn, src_dir, def_config, tmp_path, sql_preprocessor,
temp_db_cursor, threads, temp_db):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
def test_add_tiger_data_bad_file(def_config, tmp_path, sql_preprocessor,
temp_db_cursor, threads, temp_db_with_extensions):
temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.txt'
sqlfile.write_text("""Random text""")
tiger_data.add_tiger_data(dsn, str(tmp_path), threads, def_config, src_dir / 'lib-sql')
tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
assert temp_db_cursor.table_rows('place') == 0
@pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path,
temp_db_cursor, threads, temp_db, sql_preprocessor):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
def test_add_tiger_data_tarfile(def_config, tmp_path, temp_db_cursor,
threads, temp_db_with_extensions, sql_preprocessor):
temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.sql'
sqlfile.write_text("""INSERT INTO place values (1);
@@ -46,21 +42,20 @@ def test_add_tiger_data_tarfile(dsn, src_dir, def_config, tmp_path,
tar = tarfile.open("sample.tar.gz", "w:gz")
tar.add(sqlfile)
tar.close()
tiger_data.add_tiger_data(dsn, str(src_dir / 'sample.tar.gz'), threads, def_config, src_dir / 'lib-sql')
tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
assert temp_db_cursor.table_rows('place') == 1
@pytest.mark.parametrize("threads", (1, 5))
def test_add_tiger_data_bad_tarfile(dsn, src_dir, def_config, tmp_path,
temp_db_cursor, threads, temp_db, sql_preprocessor):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
def test_add_tiger_data_bad_tarfile(def_config, tmp_path, temp_db_cursor, threads,
temp_db_with_extensions, sql_preprocessor):
temp_db_cursor.execute('CREATE TABLE place (id INT)')
sqlfile = tmp_path / '1010.txt'
sqlfile.write_text("""Random text""")
tar = tarfile.open("sample.tar.gz", "w:gz")
tar.add(sqlfile)
tar.close()
tiger_data.add_tiger_data(dsn, str(src_dir / 'sample.tar.gz'), threads, def_config, src_dir / 'lib-sql')
assert temp_db_cursor.table_rows('place') == 0
tiger_data.add_tiger_data(str(tmp_path), def_config, threads)
assert temp_db_cursor.table_rows('place') == 0